From a39eea7ed6e02c0e58d70d1aff729254bd5ebb5b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 18 Jul 2026 14:13:27 +0000 Subject: [PATCH] Enforce redaction for resolved pipeline summaries --- docs/roadmap/implementation.md | 2 +- internal/cli/run.go | 2 +- internal/cli/state_hardening_test.go | 45 +++++- internal/core/config/redaction.go | 36 ++--- internal/core/config/redaction_test.go | 169 +++++++++++++++++++++++ internal/core/debugbundle/bundle_test.go | 9 +- internal/core/debugbundle/summary.go | 11 +- 7 files changed, 252 insertions(+), 22 deletions(-) create mode 100644 internal/core/config/redaction_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 6f0eb5f..c6d436b 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -156,7 +156,7 @@ exclusivity, or failure reporting. ## Stage 1: Enforce redaction for resolved-pipeline summaries -**Status:** Not started +**Status:** Complete ### Objective diff --git a/internal/cli/run.go b/internal/cli/run.go index 514fe61..12ef0f5 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -288,7 +288,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i if err := writeSummary(summary, func() error { return summary.WriteRedactedEffectiveConfig(effective) }); err != nil { return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug effective config: %w", err), false) } - if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective.ResolvedPipeline) }); err != nil { + if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil { return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug resolved pipeline: %w", err), false) } if err := writeSummary(summary, func() error { diff --git a/internal/cli/state_hardening_test.go b/internal/cli/state_hardening_test.go index e085d6d..1c8227f 100644 --- a/internal/cli/state_hardening_test.go +++ b/internal/cli/state_hardening_test.go @@ -248,6 +248,49 @@ func TestRunDebugArtifactsRedactSecretsButRetainApplicationData(t *testing.T) { } } +func TestRunRedactsSensitiveModuleOptionsFromConfigAndPipelineSummaries(t *testing.T) { + roots := newStateTestRoots(t) + data, err := os.ReadFile(roots.config) + if err != nil { + t.Fatal(err) + } + configText := strings.Replace(string(data), " input: test/input\n", ` input: + module: test/input + options: + api_key: CONFIG_SUMMARY_SECRET_SENTINEL + safe: SAFE_OPTION_SENTINEL + nested: + - - password: PIPELINE_SUMMARY_SECRET_SENTINEL + neighbor: SAFE_NESTED_OPTION_SENTINEL +`, 1) + if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil { + t.Fatal(err) + } + + result := runStateTest(t, roots, newStateTestHarness().options(), true, false, "bypass") + if result.code != 0 { + t.Fatalf("code=%d stderr=%q", result.code, result.stderr) + } + summaryRoot := filepath.Join(onlyChildDir(t, roots.debug), "summary") + for _, name := range []string{"effective-config.json", "resolved-pipeline.json"} { + contents, err := os.ReadFile(filepath.Join(summaryRoot, name)) + if err != nil { + t.Fatal(err) + } + text := string(contents) + for _, secret := range []string{"CONFIG_SUMMARY_SECRET_SENTINEL", "PIPELINE_SUMMARY_SECRET_SENTINEL"} { + if strings.Contains(text, secret) { + t.Fatalf("%s contains %q: %s", name, secret, text) + } + } + for _, retained := range []string{"[REDACTED]", "SAFE_OPTION_SENTINEL", "SAFE_NESTED_OPTION_SENTINEL"} { + if !strings.Contains(text, retained) { + t.Fatalf("%s does not contain %q: %s", name, retained, text) + } + } + } +} + type stateTestRoots struct{ config, input, output, plans, checkpoints, debug string } func newStateTestRoots(t *testing.T) stateTestRoots { @@ -422,7 +465,7 @@ func (h *stateTestHarness) options() Options { if err := pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, stateTestCodec{}); err != nil { panic(err) } - if err := registries.Inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func() (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil { + if err := registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil { panic(err) } if err := registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}}, func() (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil { diff --git a/internal/core/config/redaction.go b/internal/core/config/redaction.go index 7dfa49a..a25e1f2 100644 --- a/internal/core/config/redaction.go +++ b/internal/core/config/redaction.go @@ -25,6 +25,10 @@ func (e EffectiveConfig) RedactedSummaryPayload() any { } } +func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline { + return cloneResolvedPipeline(e.ResolvedPipeline) +} + func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline { out := in out.Input = redactBinding(cloneModuleBinding(in.Input)) @@ -116,26 +120,26 @@ func redactOptions(values map[string]any) map[string]any { out[key] = "[REDACTED]" continue } - switch typed := value.(type) { - case map[string]any: - out[key] = redactOptions(typed) - case []any: - items := make([]any, len(typed)) - for i, item := range typed { - if nested, ok := item.(map[string]any); ok { - items[i] = redactOptions(nested) - } else { - items[i] = item - } - } - out[key] = items - default: - out[key] = value - } + out[key] = redactOptionValue(value) } return out } +func redactOptionValue(value any) any { + switch typed := value.(type) { + case map[string]any: + return redactOptions(typed) + case []any: + items := make([]any, len(typed)) + for i, item := range typed { + items[i] = redactOptionValue(item) + } + return items + default: + return typed + } +} + func sensitiveConfigKey(key string) bool { key = strings.ToLower(key) return strings.Contains(key, "api_key") || strings.Contains(key, "apikey") || strings.Contains(key, "authorization") || strings.Contains(key, "bearer") || strings.Contains(key, "password") || strings.Contains(key, "secret") || strings.Contains(key, "token") diff --git a/internal/core/config/redaction_test.go b/internal/core/config/redaction_test.go new file mode 100644 index 0000000..65aa993 --- /dev/null +++ b/internal/core/config/redaction_test.go @@ -0,0 +1,169 @@ +package config + +import ( + "encoding/json" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +func TestRedactedResolvedPipelinePayloadRedactsEveryBinding(t *testing.T) { + bindings := map[string]pipeline.ModuleBinding{} + for _, name := range []string{ + "input", "chunk", "output", "extract", "merge", "normalize", + "resolved-validator", "lane-validator", + } { + bindings[name] = redactionTestBinding(name) + } + + resolved := pipeline.ResolvedPipeline{ + ID: "redaction-test", + Digest: "sha256:safe-digest", + Input: bindings["input"], + Chunk: bindings["chunk"], + ChunkReferences: redactionTestReferenceTarget(pipeline.StageChunk, "", "chunk-reference-content"), + ArtifactLanes: []pipeline.ResolvedArtifactLane{{ + ID: "safe-lane", + ArtifactKind: "safe/artifact", + Extract: bindings["extract"], + Merge: bindings["merge"], + Normalize: bindings["normalize"], + Validators: []pipeline.ModuleBinding{bindings["lane-validator"]}, + ExtractReferences: redactionTestReferenceTarget(pipeline.StageExtract, "safe-lane", "extract-reference-content"), + MergeReferences: redactionTestReferenceTarget(pipeline.StageMerge, "safe-lane", "merge-reference-content"), + NormalizeReferences: redactionTestReferenceTarget(pipeline.StageNormalize, "safe-lane", "normalize-reference-content"), + }}, + ValidatorChains: []pipeline.ResolvedValidatorChain{{ + Stage: pipeline.StageExtract, + LaneID: "safe-lane", + ModuleKey: "safe-extract-owner", + Validators: []pipeline.ResolvedValidator{{ + Binding: bindings["resolved-validator"], + ExecutionClass: contracts.ExecutionClassDeterministic, + Target: pipeline.ValidatorTargetTyped, + ArtifactKind: "safe/artifact", + }}, + }}, + Output: bindings["output"], + } + effective := EffectiveConfig{ + Config: Config{Pipelines: map[string]pipeline.PipelineProfile{ + "redaction-test": {Input: bindings["input"]}, + }}, + PipelineID: "redaction-test", + ResolvedPipeline: resolved, + } + + payload := effective.RedactedResolvedPipelinePayload() + encoded, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + text := string(encoded) + for name := range bindings { + for _, forbidden := range []string{name + "-secret", name + "-nested-secret"} { + if strings.Contains(text, forbidden) { + t.Fatalf("resolved pipeline summary contains %q: %s", forbidden, text) + } + } + if !strings.Contains(text, name+"-safe") { + t.Fatalf("resolved pipeline summary does not retain safe option for %q: %s", name, text) + } + } + for _, content := range []string{ + "chunk-reference-content", "extract-reference-content", + "merge-reference-content", "normalize-reference-content", + } { + if strings.Contains(text, content) { + t.Fatalf("resolved pipeline summary contains materialized reference content %q", content) + } + } + for _, safe := range []string{"[REDACTED]", "safe-reference-path", "safe-binding-source"} { + if !strings.Contains(text, safe) { + t.Fatalf("resolved pipeline summary does not retain %q: %s", safe, text) + } + } + + payload.Input.Options["safe"] = "mutated" + nested := payload.Input.Options["nested"].([]any)[0].([]any)[0].(map[string]any) + nested["neighbor"] = "mutated" + payload.ChunkReferences.ReferenceSet.Slots["safe-slot"].Items[0].Content[0] = 'X' + payload.ValidatorChains[0].Validators[0].Binding.Options["safe"] = "mutated" + + assertRedactionTestBindingUnchanged(t, effective.ResolvedPipeline.Input, "input") + assertRedactionTestBindingUnchanged(t, effective.Config.Pipelines["redaction-test"].Input, "input") + assertRedactionTestBindingUnchanged(t, effective.ResolvedPipeline.ValidatorChains[0].Validators[0].Binding, "resolved-validator") + if got := string(effective.ResolvedPipeline.ChunkReferences.ReferenceSet.Slots["safe-slot"].Items[0].Content); got != "chunk-reference-content" { + t.Fatalf("source reference content mutated through redacted payload: %q", got) + } +} + +func TestRedactedEffectiveConfigPayloadDoesNotAliasSource(t *testing.T) { + binding := redactionTestBinding("effective") + effective := EffectiveConfig{ + Config: Config{Pipelines: map[string]pipeline.PipelineProfile{ + "redaction-test": {Input: binding}, + }}, + ResolvedPipeline: pipeline.ResolvedPipeline{Input: binding}, + } + + payload := effective.RedactedSummaryPayload().(EffectiveConfig) + payload.Config.Pipelines["redaction-test"].Input.Options["safe"] = "mutated" + payload.ResolvedPipeline.Input.Options["safe"] = "mutated" + + assertRedactionTestBindingUnchanged(t, effective.Config.Pipelines["redaction-test"].Input, "effective") + assertRedactionTestBindingUnchanged(t, effective.ResolvedPipeline.Input, "effective") +} + +func redactionTestBinding(name string) pipeline.ModuleBinding { + return pipeline.ModuleBinding{ + Module: "safe-" + name, + Options: map[string]any{ + "api_key": name + "-secret", + "safe": name + "-safe", + "nested": []any{[]any{map[string]any{ + "password": name + "-nested-secret", + "neighbor": name + "-nested-safe", + }}}, + }, + } +} + +func redactionTestReferenceTarget(stage pipeline.ModuleStage, laneID, content string) pipeline.ResolvedReferenceTarget { + return pipeline.ResolvedReferenceTarget{ + Stage: stage, + LaneID: laneID, + Module: "safe-reference-module", + Bindings: []pipeline.ReferenceBinding{{ + Stage: stage, + LaneID: laneID, + SlotName: "safe-slot", + Source: "safe-reference-path", + BindingSource: "safe-binding-source", + }}, + ReferenceSet: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{ + "safe-slot": { + Slot: contracts.ReferenceSlot{Name: "safe-slot"}, + Items: []contracts.ReferenceItem{{ + SlotName: "safe-slot", + Content: []byte(content), + Digest: "sha256:safe-reference-digest", + BindingSource: "safe-binding-source", + }}, + }, + }}, + } +} + +func assertRedactionTestBindingUnchanged(t *testing.T, binding pipeline.ModuleBinding, name string) { + t.Helper() + if got := binding.Options["safe"]; got != name+"-safe" { + t.Fatalf("source safe option = %v, want %q", got, name+"-safe") + } + nested := binding.Options["nested"].([]any)[0].([]any)[0].(map[string]any) + if got := nested["neighbor"]; got != name+"-nested-safe" { + t.Fatalf("source nested safe option = %v, want %q", got, name+"-nested-safe") + } +} diff --git a/internal/core/debugbundle/bundle_test.go b/internal/core/debugbundle/bundle_test.go index 6ca7646..46498af 100644 --- a/internal/core/debugbundle/bundle_test.go +++ b/internal/core/debugbundle/bundle_test.go @@ -8,6 +8,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) func TestAllocateCreatesRestrictiveSummaryAndTrace(t *testing.T) { @@ -92,7 +93,7 @@ func TestSummaryWriterWritesEverySummaryArtifact(t *testing.T) { if err := summary.WriteRedactedEffectiveConfig(testRedactedSummaryPayload{}); err != nil { t.Fatal(err) } - if err := summary.WriteResolvedPipeline(map[string]any{}); err != nil { + if err := summary.WriteResolvedPipeline(testRedactedResolvedPipelinePayload{}); err != nil { t.Fatal(err) } if err := summary.WriteResolvedReferences(nil); err != nil { @@ -156,3 +157,9 @@ type testRedactedSummaryPayload struct{} func (testRedactedSummaryPayload) RedactedSummaryPayload() any { return map[string]any{"redacted": true} } + +type testRedactedResolvedPipelinePayload struct{} + +func (testRedactedResolvedPipelinePayload) RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline { + return pipeline.ResolvedPipeline{ID: "redacted"} +} diff --git a/internal/core/debugbundle/summary.go b/internal/core/debugbundle/summary.go index b4ca219..d596c1c 100644 --- a/internal/core/debugbundle/summary.go +++ b/internal/core/debugbundle/summary.go @@ -8,6 +8,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/fileio" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) const ( @@ -24,6 +25,9 @@ const ( ) type RedactedSummaryPayload interface{ RedactedSummaryPayload() any } +type RedactedResolvedPipelinePayload interface { + RedactedResolvedPipelinePayload() pipeline.ResolvedPipeline +} type Invocation struct { Operation string `json:"operation"` PipelineID string `json:"pipeline_id,omitempty"` @@ -71,8 +75,11 @@ func (w *SummaryWriter) WriteRedactedEffectiveConfig(payload RedactedSummaryPayl } return w.WriteJSON(ArtifactEffectiveConfig, payload.RedactedSummaryPayload()) } -func (w *SummaryWriter) WriteResolvedPipeline(v any) error { - return w.WriteJSON(ArtifactResolvedPipeline, v) +func (w *SummaryWriter) WriteResolvedPipeline(payload RedactedResolvedPipelinePayload) error { + if payload == nil { + return fmt.Errorf("redacted resolved pipeline payload must not be nil") + } + return w.WriteJSON(ArtifactResolvedPipeline, payload.RedactedResolvedPipelinePayload()) } func (w *SummaryWriter) WriteResolvedReferences(v any) error { return w.WriteJSON(ArtifactResolvedReferences, v)