Enforce redaction for resolved pipeline summaries

2026-07-18 14:13:27 +00:00
parent 7bcce9953e
commit a39eea7ed6
7 changed files with 252 additions and 22 deletions
--- a/docs/roadmap/implementation.md
+++ b/docs/roadmap/implementation.md
@@ -156,7 +156,7 @@ exclusivity, or failure reporting.

 ## Stage 1: Enforce redaction for resolved-pipeline summaries

-**Status:** Not started
+**Status:** Complete

 ### Objective

--- a/internal/cli/run.go
+++ b/internal/cli/run.go
@@ -288,7 +288,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
 	if err := writeSummary(summary, func() error { return summary.WriteRedactedEffectiveConfig(effective) }); err != nil {
 		return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug effective config: %w", err), false)
 	}
-	if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective.ResolvedPipeline) }); err != nil {
+	if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil {
 		return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug resolved pipeline: %w", err), false)
 	}
 	if err := writeSummary(summary, func() error {
--- a/internal/cli/state_hardening_test.go
+++ b/internal/cli/state_hardening_test.go
@@ -248,6 +248,49 @@ func TestRunDebugArtifactsRedactSecretsButRetainApplicationData(t *testing.T) {
 	}
 }

+func TestRunRedactsSensitiveModuleOptionsFromConfigAndPipelineSummaries(t *testing.T) {
+	roots := newStateTestRoots(t)
+	data, err := os.ReadFile(roots.config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	configText := strings.Replace(string(data), "    input: test/input\n", `    input:
+      module: test/input
+      options:
+        api_key: CONFIG_SUMMARY_SECRET_SENTINEL
+        safe: SAFE_OPTION_SENTINEL
+        nested:
+          - - password: PIPELINE_SUMMARY_SECRET_SENTINEL
+              neighbor: SAFE_NESTED_OPTION_SENTINEL
+`, 1)
+	if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	result := runStateTest(t, roots, newStateTestHarness().options(), true, false, "bypass")
+	if result.code != 0 {
+		t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
+	}
+	summaryRoot := filepath.Join(onlyChildDir(t, roots.debug), "summary")
+	for _, name := range []string{"effective-config.json", "resolved-pipeline.json"} {
+		contents, err := os.ReadFile(filepath.Join(summaryRoot, name))
+		if err != nil {
+			t.Fatal(err)
+		}
+		text := string(contents)
+		for _, secret := range []string{"CONFIG_SUMMARY_SECRET_SENTINEL", "PIPELINE_SUMMARY_SECRET_SENTINEL"} {
+			if strings.Contains(text, secret) {
+				t.Fatalf("%s contains %q: %s", name, secret, text)
+			}
+		}
+		for _, retained := range []string{"[REDACTED]", "SAFE_OPTION_SENTINEL", "SAFE_NESTED_OPTION_SENTINEL"} {
+			if !strings.Contains(text, retained) {
+				t.Fatalf("%s does not contain %q: %s", name, retained, text)
+			}
+		}
+	}
+}
+
 type stateTestRoots struct{ config, input, output, plans, checkpoints, debug string }

 func newStateTestRoots(t *testing.T) stateTestRoots {
@@ -422,7 +465,7 @@ func (h *stateTestHarness) options() Options {
 	if err := pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, stateTestCodec{}); err != nil {
 		panic(err)
 	}
-	if err := registries.Inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func() (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil {
+	if err := registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil {
 		panic(err)
 	}
 	if err := registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}}, func() (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil {
--- a/internal/core/config/redaction.go
+++ b/internal/core/config/redaction.go
@@ -25,6 +25,10 @@ func (e EffectiveConfig) RedactedSummaryPayload() any {
 	}
 }

+func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline {
+	return cloneResolvedPipeline(e.ResolvedPipeline)
+}
+
 func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline {
 	out := in
 	out.Input = redactBinding(cloneModuleBinding(in.Input))
@@ -116,26 +120,26 @@ func redactOptions(values map[string]any) map[string]any {
 			out[key] = "[REDACTED]"
 			continue
 		}
-		switch typed := value.(type) {
-		case map[string]any:
-			out[key] = redactOptions(typed)
-		case []any:
-			items := make([]any, len(typed))
-			for i, item := range typed {
-				if nested, ok := item.(map[string]any); ok {
-					items[i] = redactOptions(nested)
-				} else {
-					items[i] = item
-				}
-			}
-			out[key] = items
-		default:
-			out[key] = value
-		}
+		out[key] = redactOptionValue(value)
 	}
 	return out
 }

+func redactOptionValue(value any) any {
+	switch typed := value.(type) {
+	case map[string]any:
+		return redactOptions(typed)
+	case []any:
+		items := make([]any, len(typed))
+		for i, item := range typed {
+			items[i] = redactOptionValue(item)
+		}
+		return items
+	default:
+		return typed
+	}
+}
+
 func sensitiveConfigKey(key string) bool {
 	key = strings.ToLower(key)
 	return strings.Contains(key, "api_key") || strings.Contains(key, "apikey") || strings.Contains(key, "authorization") || strings.Contains(key, "bearer") || strings.Contains(key, "password") || strings.Contains(key, "secret") || strings.Contains(key, "token")
--- a/internal/core/config/redaction_test.go
+++ b/internal/core/config/redaction_test.go
@@ -0,0 +1,169 @@
+package config
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestRedactedResolvedPipelinePayloadRedactsEveryBinding(t *testing.T) {
+	bindings := map[string]pipeline.ModuleBinding{}
+	for _, name := range []string{
+		"input", "chunk", "output", "extract", "merge", "normalize",
+		"resolved-validator", "lane-validator",
+	} {
+		bindings[name] = redactionTestBinding(name)
+	}
+
+	resolved := pipeline.ResolvedPipeline{
+		ID:              "redaction-test",
+		Digest:          "sha256:safe-digest",
+		Input:           bindings["input"],
+		Chunk:           bindings["chunk"],
+		ChunkReferences: redactionTestReferenceTarget(pipeline.StageChunk, "", "chunk-reference-content"),
+		ArtifactLanes: []pipeline.ResolvedArtifactLane{{
+			ID:                  "safe-lane",
+			ArtifactKind:        "safe/artifact",
+			Extract:             bindings["extract"],
+			Merge:               bindings["merge"],
+			Normalize:           bindings["normalize"],
+			Validators:          []pipeline.ModuleBinding{bindings["lane-validator"]},
+			ExtractReferences:   redactionTestReferenceTarget(pipeline.StageExtract, "safe-lane", "extract-reference-content"),
+			MergeReferences:     redactionTestReferenceTarget(pipeline.StageMerge, "safe-lane", "merge-reference-content"),
+			NormalizeReferences: redactionTestReferenceTarget(pipeline.StageNormalize, "safe-lane", "normalize-reference-content"),
+		}},
+		ValidatorChains: []pipeline.ResolvedValidatorChain{{
+			Stage:     pipeline.StageExtract,
+			LaneID:    "safe-lane",
+			ModuleKey: "safe-extract-owner",
+			Validators: []pipeline.ResolvedValidator{{
+				Binding:        bindings["resolved-validator"],
+				ExecutionClass: contracts.ExecutionClassDeterministic,
+				Target:         pipeline.ValidatorTargetTyped,
+				ArtifactKind:   "safe/artifact",
+			}},
+		}},
+		Output: bindings["output"],
+	}
+	effective := EffectiveConfig{
+		Config: Config{Pipelines: map[string]pipeline.PipelineProfile{
+			"redaction-test": {Input: bindings["input"]},
+		}},
+		PipelineID:       "redaction-test",
+		ResolvedPipeline: resolved,
+	}
+
+	payload := effective.RedactedResolvedPipelinePayload()
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+	text := string(encoded)
+	for name := range bindings {
+		for _, forbidden := range []string{name + "-secret", name + "-nested-secret"} {
+			if strings.Contains(text, forbidden) {
+				t.Fatalf("resolved pipeline summary contains %q: %s", forbidden, text)
+			}
+		}
+		if !strings.Contains(text, name+"-safe") {
+			t.Fatalf("resolved pipeline summary does not retain safe option for %q: %s", name, text)
+		}
+	}
+	for _, content := range []string{
+		"chunk-reference-content", "extract-reference-content",
+		"merge-reference-content", "normalize-reference-content",
+	} {
+		if strings.Contains(text, content) {
+			t.Fatalf("resolved pipeline summary contains materialized reference content %q", content)
+		}
+	}
+	for _, safe := range []string{"[REDACTED]", "safe-reference-path", "safe-binding-source"} {
+		if !strings.Contains(text, safe) {
+			t.Fatalf("resolved pipeline summary does not retain %q: %s", safe, text)
+		}
+	}
+
+	payload.Input.Options["safe"] = "mutated"
+	nested := payload.Input.Options["nested"].([]any)[0].([]any)[0].(map[string]any)
+	nested["neighbor"] = "mutated"
+	payload.ChunkReferences.ReferenceSet.Slots["safe-slot"].Items[0].Content[0] = 'X'
+	payload.ValidatorChains[0].Validators[0].Binding.Options["safe"] = "mutated"
+
+	assertRedactionTestBindingUnchanged(t, effective.ResolvedPipeline.Input, "input")
+	assertRedactionTestBindingUnchanged(t, effective.Config.Pipelines["redaction-test"].Input, "input")
+	assertRedactionTestBindingUnchanged(t, effective.ResolvedPipeline.ValidatorChains[0].Validators[0].Binding, "resolved-validator")
+	if got := string(effective.ResolvedPipeline.ChunkReferences.ReferenceSet.Slots["safe-slot"].Items[0].Content); got != "chunk-reference-content" {
+		t.Fatalf("source reference content mutated through redacted payload: %q", got)
+	}
+}
+
+func TestRedactedEffectiveConfigPayloadDoesNotAliasSource(t *testing.T) {
+	binding := redactionTestBinding("effective")
+	effective := EffectiveConfig{
+		Config: Config{Pipelines: map[string]pipeline.PipelineProfile{
+			"redaction-test": {Input: binding},
+		}},
+		ResolvedPipeline: pipeline.ResolvedPipeline{Input: binding},
+	}
+
+	payload := effective.RedactedSummaryPayload().(EffectiveConfig)
+	payload.Config.Pipelines["redaction-test"].Input.Options["safe"] = "mutated"
+	payload.ResolvedPipeline.Input.Options["safe"] = "mutated"
+
+	assertRedactionTestBindingUnchanged(t, effective.Config.Pipelines["redaction-test"].Input, "effective")
+	assertRedactionTestBindingUnchanged(t, effective.ResolvedPipeline.Input, "effective")
+}
+
+func redactionTestBinding(name string) pipeline.ModuleBinding {
+	return pipeline.ModuleBinding{
+		Module: "safe-" + name,
+		Options: map[string]any{
+			"api_key": name + "-secret",
+			"safe":    name + "-safe",
+			"nested": []any{[]any{map[string]any{
+				"password": name + "-nested-secret",
+				"neighbor": name + "-nested-safe",
+			}}},
+		},
+	}
+}
+
+func redactionTestReferenceTarget(stage pipeline.ModuleStage, laneID, content string) pipeline.ResolvedReferenceTarget {
+	return pipeline.ResolvedReferenceTarget{
+		Stage:  stage,
+		LaneID: laneID,
+		Module: "safe-reference-module",
+		Bindings: []pipeline.ReferenceBinding{{
+			Stage:         stage,
+			LaneID:        laneID,
+			SlotName:      "safe-slot",
+			Source:        "safe-reference-path",
+			BindingSource: "safe-binding-source",
+		}},
+		ReferenceSet: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
+			"safe-slot": {
+				Slot: contracts.ReferenceSlot{Name: "safe-slot"},
+				Items: []contracts.ReferenceItem{{
+					SlotName:      "safe-slot",
+					Content:       []byte(content),
+					Digest:        "sha256:safe-reference-digest",
+					BindingSource: "safe-binding-source",
+				}},
+			},
+		}},
+	}
+}
+
+func assertRedactionTestBindingUnchanged(t *testing.T, binding pipeline.ModuleBinding, name string) {
+	t.Helper()
+	if got := binding.Options["safe"]; got != name+"-safe" {
+		t.Fatalf("source safe option = %v, want %q", got, name+"-safe")
+	}
+	nested := binding.Options["nested"].([]any)[0].([]any)[0].(map[string]any)
+	if got := nested["neighbor"]; got != name+"-nested-safe" {
+		t.Fatalf("source nested safe option = %v, want %q", got, name+"-nested-safe")
+	}
+}
--- a/internal/core/debugbundle/bundle_test.go
+++ b/internal/core/debugbundle/bundle_test.go
@@ -8,6 +8,7 @@ import (

 	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
 )

 func TestAllocateCreatesRestrictiveSummaryAndTrace(t *testing.T) {
@@ -92,7 +93,7 @@ func TestSummaryWriterWritesEverySummaryArtifact(t *testing.T) {
 	if err := summary.WriteRedactedEffectiveConfig(testRedactedSummaryPayload{}); err != nil {
 		t.Fatal(err)
 	}
-	if err := summary.WriteResolvedPipeline(map[string]any{}); err != nil {
+	if err := summary.WriteResolvedPipeline(testRedactedResolvedPipelinePayload{}); err != nil {
 		t.Fatal(err)
 	}
 	if err := summary.WriteResolvedReferences(nil); err != nil {
@@ -156,3 +157,9 @@ type testRedactedSummaryPayload struct{}
 func (testRedactedSummaryPayload) RedactedSummaryPayload() any {
 	return map[string]any{"redacted": true}
 }
+
+type testRedactedResolvedPipelinePayload struct{}
+
+func (testRedactedResolvedPipelinePayload) RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline {
+	return pipeline.ResolvedPipeline{ID: "redacted"}
+}
--- a/internal/core/debugbundle/summary.go
+++ b/internal/core/debugbundle/summary.go
@@ -8,6 +8,7 @@ import (
 	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
 	"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
 )

 const (
@@ -24,6 +25,9 @@ const (
 )

 type RedactedSummaryPayload interface{ RedactedSummaryPayload() any }
+type RedactedResolvedPipelinePayload interface {
+	RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline
+}
 type Invocation struct {
 	Operation          string    `json:"operation"`
 	PipelineID         string    `json:"pipeline_id,omitempty"`
@@ -71,8 +75,11 @@ func (w *SummaryWriter) WriteRedactedEffectiveConfig(payload RedactedSummaryPayl
 	}
 	return w.WriteJSON(ArtifactEffectiveConfig, payload.RedactedSummaryPayload())
 }
-func (w *SummaryWriter) WriteResolvedPipeline(v any) error {
-	return w.WriteJSON(ArtifactResolvedPipeline, v)
+func (w *SummaryWriter) WriteResolvedPipeline(payload RedactedResolvedPipelinePayload) error {
+	if payload == nil {
+		return fmt.Errorf("redacted resolved pipeline payload must not be nil")
+	}
+	return w.WriteJSON(ArtifactResolvedPipeline, payload.RedactedResolvedPipelinePayload())
 }
 func (w *SummaryWriter) WriteResolvedReferences(v any) error {
 	return w.WriteJSON(ArtifactResolvedReferences, v)