Record configuration normalization at sequence boundaries

internal/core/config/document_normalizer.go

@@ -0,0 +1,26 @@
+package config
+
+import configkit "gitea.maximumdirect.net/eric/ConfigKit"
+
+// documentNormalizer records leaf provenance and, while visiting a sequence,
+// covers its atomic mutation boundary. ConfigKit audits sequences as whole
+// values, so a record for only a descendant is insufficient. The outermost
+// sequence covers nested sequences as well; unchanged visits record nothing.
+type documentNormalizer struct {
+	*configkit.Resolution
+	sequence *configkit.Path
+}
+
+func (n documentNormalizer) inSequence(path configkit.Path) documentNormalizer {
+	if n.sequence == nil {
+		n.sequence = &path
+	}
+	return n
+}
+
+func (n documentNormalizer) Record(path configkit.Path, kind configkit.ChangeKind, reason string, dependencies ...configkit.Path) {
+	n.Resolution.Record(path, kind, reason, dependencies...)
+	if n.sequence != nil {
+		n.Resolution.Record(*n.sequence, kind, reason, append([]configkit.Path{path}, dependencies...)...)
+	}
+}

internal/core/config/document_paths.go

@@ -6,7 +6,8 @@ import (
 	configkit "gitea.maximumdirect.net/eric/ConfigKit"
 )
 
-func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
+func normalizeDocumentPaths(document *Document, context *configkit.Resolution, state *documentLoadState) {
+	resolution := documentNormalizer{Resolution: context}
 	if document == nil {
 		return
 	}
@@ -40,19 +41,17 @@ func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution
 		if profile.Output != nil {
 			normalizeDocumentBindingReferencePaths(profile.Output, path.Key("output"), resolution, state)
 		}
-		if documentStepsContainRelativeReferencePaths(profile.Steps) {
-			resolution.Record(path.Key("steps"), configkit.ChangeNormalization, "resolved configured paths in ordered steps")
-		}
 		for laneID, lane := range profile.Artifacts {
 			normalizeDocumentLaneReferencePaths(&lane, path.Key("artifacts").Key(laneID), resolution, state)
 			profile.Artifacts[laneID] = lane
 		}
+		stepResolution := resolution.inSequence(path.Key("steps"))
 		for index := range profile.Steps {
 			step := &profile.Steps[index]
 			stepPath := path.Key("steps").Index(index)
-			normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), resolution, state)
+			normalizeDocumentReferencePaths(step.References, stepPath.Key("references"), stepResolution, state)
 			for laneID, lane := range step.Artifacts {
-				normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), resolution, state)
+				normalizeDocumentLaneReferencePaths(&lane, stepPath.Key("artifacts").Key(laneID), stepResolution, state)
 				step.Artifacts[laneID] = lane
 			}
 		}
@@ -60,32 +59,7 @@ func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution
 	}
 }
 
-func documentStepsContainRelativeReferencePaths(steps []PipelineStepDocument) bool {
-	for _, step := range steps {
-		if documentReferencesContainRelativePath(step.References) {
-			return true
-		}
-		for _, lane := range step.Artifacts {
-			if documentReferencesContainRelativePath(lane.Extract.References) ||
-				(lane.Merge != nil && documentReferencesContainRelativePath(lane.Merge.References)) ||
-				(lane.Normalize != nil && documentReferencesContainRelativePath(lane.Normalize.References)) {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-func documentReferencesContainRelativePath(references map[string]ReferenceDocument) bool {
-	for _, source := range references {
-		if source.Path != nil && *source.Path != "" && !filepath.IsAbs(*source.Path) {
-			return true
-		}
-	}
-	return false
-}
-
-func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *documentLoadState) {
+func initializeDocumentMainDirectory(resolution documentNormalizer, state *documentLoadState) {
 	if state == nil || state.mainDirectory != "" {
 		return
 	}
@@ -102,13 +76,13 @@ func initializeDocumentMainDirectory(resolution *configkit.Resolution, state *do
 	state.mainDirectory = cleanInvocationDirectory(".")
 }
 
-func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
+func normalizeDocumentBindingReferencePaths(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
 	if binding != nil {
 		normalizeDocumentReferencePaths(binding.References, path.Key("references"), resolution, state)
 	}
 }
 
-func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
+func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
 	if lane == nil {
 		return
 	}
@@ -117,7 +91,7 @@ func normalizeDocumentLaneReferencePaths(lane *ArtifactLaneDocument, path config
 	normalizeDocumentBindingReferencePaths(lane.Normalize, path.Key("normalize"), resolution, state)
 }
 
-func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
+func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
 	for _, name := range sortedReferenceDocumentKeys(references) {
 		source := references[name]
 		sourcePath := path.Key(name).Key("path")
@@ -137,7 +111,7 @@ func normalizeDocumentReferencePaths(references map[string]ReferenceDocument, pa
 	}
 }
 
-func normalizeDocumentPath(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
+func normalizeDocumentPath(resolution documentNormalizer, state *documentLoadState, path configkit.Path, target **string, optional bool, reason string) {
 	if target == nil || *target == nil || **target == "" || filepath.IsAbs(**target) {
 		return
 	}
@@ -151,11 +125,11 @@ func normalizeDocumentPath(resolution *configkit.Resolution, state *documentLoad
 		return
 	}
 	if *target != nil && **target != value {
-		configkit.Normalize(resolution, path, target, documentValue(value), reason)
+		configkit.Normalize(resolution.Resolution, path, target, documentValue(value), reason)
 	}
 }
 
-func documentPathBase(resolution *configkit.Resolution, state *documentLoadState, path configkit.Path) (string, bool) {
+func documentPathBase(resolution documentNormalizer, state *documentLoadState, path configkit.Path) (string, bool) {
 	original := path
 	if state != nil {
 		original = state.originalPath(path)

internal/core/config/pipeline_document.go

@@ -64,14 +64,15 @@ func normalizeDocumentPipelines(document *Document, resolution *configkit.Resolu
 		return
 	}
 	validateDocumentPipelinePresence(document, resolution)
+	normalizer := documentNormalizer{Resolution: resolution}
 
 	pipelinesPath := configkit.Keys("pipelines")
 	for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
 		profile := document.Pipelines[id]
-		normalizePipelineDocument(&profile, pipelinesPath.Key(id), resolution, state)
+		normalizePipelineDocument(&profile, pipelinesPath.Key(id), normalizer, state)
 		document.Pipelines[id] = profile
 	}
-	document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", resolution, state)
+	document.Pipelines = normalizeDocumentMap(document.Pipelines, pipelinesPath, "pipeline identifier", normalizer, state)
 }
 
 func validateDocumentPipelinePresence(document *Document, resolution *configkit.Resolution) {
@@ -175,7 +176,7 @@ func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path confi
 	rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil)
 }
 
-func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
+func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
 	normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile")
 	normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state)
 	if profile.Chunk != nil {
@@ -191,6 +192,7 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
 		profile.Artifacts[laneID] = lane
 	}
 	profile.Artifacts = normalizeDocumentMap(profile.Artifacts, path.Key("artifacts"), "artifact lane identifier", resolution, state)
+	resolution = resolution.inSequence(path.Key("steps"))
 	for index := range profile.Steps {
 		step := &profile.Steps[index]
 		stepPath := path.Key("steps").Index(index)
@@ -205,7 +207,7 @@ func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, r
 	}
 }
 
-func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
+func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
 	normalizeDocumentBinding(&lane.Extract, path.Key("extract"), resolution, state)
 	if lane.Merge != nil {
 		normalizeDocumentBinding(lane.Merge, path.Key("merge"), resolution, state)
@@ -213,26 +215,28 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa
 	if lane.Normalize != nil {
 		normalizeDocumentBinding(lane.Normalize, path.Key("normalize"), resolution, state)
 	}
+	resolution = resolution.inSequence(path.Key("validators"))
 	for index := range lane.Validators {
 		normalizeValidatorDocument(&lane.Validators[index], path.Key("validators").Index(index), resolution)
 	}
 }
 
-func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) {
+func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
 	normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
 	normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
 	binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
+	resolution = resolution.inSequence(path.Key("validators"))
 	for index := range binding.Validators {
 		normalizeValidatorDocument(&binding.Validators[index], path.Key("validators").Index(index), resolution)
 	}
 }
 
-func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution *configkit.Resolution) {
+func normalizeValidatorDocument(validator *ValidatorDocument, path configkit.Path, resolution documentNormalizer) {
 	normalizeDocumentValue(resolution, path.Key("module"), &validator.Module, strings.TrimSpace(validator.Module), "trimmed validator module identifier")
 	normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &validator.LLMProfile, "trimmed validator LLM profile")
 }
 
-func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution *configkit.Resolution, state *documentLoadState) map[string]ReferenceDocument {
+func normalizeDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) map[string]ReferenceDocument {
 	for _, name := range sortedReferenceDocumentKeys(references) {
 		source := references[name]
 		sourcePath := path.Key(name)
@@ -246,7 +250,7 @@ func normalizeDocumentReferences(references map[string]ReferenceDocument, path c
 	return normalizeDocumentMap(references, path, "reference name", resolution, state)
 }
 
-func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path configkit.Path, target *T, value T, reason string) {
+func normalizeDocumentValue[T comparable](resolution documentNormalizer, path configkit.Path, target *T, value T, reason string) {
 	if target == nil || !resolution.IsExplicit(path) {
 		return
 	}
@@ -256,7 +260,7 @@ func normalizeDocumentValue[T comparable](resolution *configkit.Resolution, path
 	}
 }
 
-func normalizePipelineDocumentString(resolution *configkit.Resolution, path configkit.Path, target **string, reason string) {
+func normalizePipelineDocumentString(resolution documentNormalizer, path configkit.Path, target **string, reason string) {
 	if target == nil || *target == nil || !resolution.IsExplicit(path) {
 		return
 	}
@@ -267,7 +271,7 @@ func normalizePipelineDocumentString(resolution *configkit.Resolution, path conf
 	}
 }
 
-func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution *configkit.Resolution, state *documentLoadState) map[string]T {
+func normalizeDocumentMap[T any](values map[string]T, path configkit.Path, kind string, resolution documentNormalizer, state *documentLoadState) map[string]T {
 	if len(values) == 0 {
 		return values
 	}

internal/core/config/sequence_normalization_test.go

@@ -0,0 +1,92 @@
+package config
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestSequenceNormalizationPreservesRuntimeAndProvenance(t *testing.T) {
+	t.Parallel()
+	for _, test := range []struct {
+		name, body, pointer, want string
+	}{
+		{
+			name:    "step identifier",
+			body:    "steps:\n      - id: ' first '\n        artifacts: {spells: {extract: {module: dnd/spells}}}",
+			pointer: "/pipelines/session/steps/0/id",
+			want:    "first",
+		},
+		{
+			name:    "standalone validator",
+			body:    "artifacts:\n      spells:\n        extract:\n          module: dnd/spells\n          validators: [{module: ' generic/valid_json '}]",
+			pointer: "/pipelines/session/artifacts/spells/extract/validators/0/module",
+			want:    "generic/valid_json",
+		},
+		{
+			name: "nested fields and source paths",
+			body: `steps:
+      - id: first
+        artifacts:
+          " spells ":
+            extract:
+              module: " dnd/spells "
+              llm_profile: " local "
+              validators: [{module: " generic/valid_json "}]
+              references:
+                " party ": {path: "./party.txt"}
+                registry: {artifact: {step: " previous ", lane: " nouns "}}`,
+			// Map-key moves are explained at their enclosing container, where
+			// the original declaration and the new canonical keys coexist.
+			pointer: "/pipelines/session/steps",
+			want:    "party.txt",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			root := t.TempDir()
+			path := filepath.Join(root, "config.yml")
+			if err := os.WriteFile(path, []byte("version: 5\npipelines:\n  session:\n    input: {module: seriatim}\n    "+test.body+"\n"), 0o600); err != nil {
+				t.Fatal(err)
+			}
+			loaded, err := Load(path, LoadOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			profile := loaded.Config.Pipelines["session"]
+			switch test.name {
+			case "step identifier":
+				if profile.Steps[0].ID != "first" {
+					t.Fatalf("step = %#v", profile.Steps[0])
+				}
+			case "standalone validator":
+				if profile.Artifacts["spells"].Extract.Validators.Validators[0].Module != test.want {
+					t.Fatal("validator was not normalized")
+				}
+			default:
+				binding := profile.Steps[0].Artifacts["spells"].Extract
+				if binding.Module != "dnd/spells" || binding.LLMProfile != "local" ||
+					binding.Validators.Validators[0].Module != "generic/valid_json" ||
+					binding.References["party"].Path != filepath.Join(root, "party.txt") ||
+					binding.References["registry"].Artifact.Step != "previous" ||
+					binding.References["registry"].Artifact.Lane != "nouns" {
+					t.Fatalf("binding = %#v", binding)
+				}
+			}
+			var effective, explanation bytes.Buffer
+			if err := loaded.WriteEffective(&effective); err != nil {
+				t.Fatal(err)
+			}
+			if err := loaded.WriteExplanation(&explanation, test.pointer); err != nil {
+				t.Fatal(err)
+			}
+			if !strings.Contains(effective.String(), test.want) ||
+				!strings.Contains(explanation.String(), test.want) ||
+				!strings.Contains(explanation.String(), path) ||
+				!strings.Contains(explanation.String(), "normalization") {
+				t.Fatalf("missing effective value or provenance:\n%s\n%s", effective.String(), explanation.String())
+			}
+		})
+	}
+}