Write terminal debug reports for failed runs

2026-07-18 14:31:46 +00:00
parent 2111e01142
commit e4ec521bed
6 changed files with 370 additions and 48 deletions
--- a/internal/cli/state_hardening_test.go
+++ b/internal/cli/state_hardening_test.go
@@ -16,6 +16,7 @@ import (

 	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
 	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
+	"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
 	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -321,6 +322,188 @@ func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) {
 	if manifest.RunID != runID {
 		t.Fatalf("manifest run ID = %q, want %q", manifest.RunID, runID)
 	}
+	report := readStateTestRunReport(t, debugPath)
+	if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" {
+		t.Fatalf("success report = %#v", report)
+	}
+	if !strings.Contains(result.stdout, "outputs=1 rejected=0") {
+		t.Fatalf("stdout=%q, want report counts", result.stdout)
+	}
+}
+
+func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *testing.T) {
+	for _, tc := range []struct {
+		name             string
+		pipelineID       string
+		wantError        string
+		wantOutputs      int
+		wantValidation   string
+		configureFailure func(*testing.T, stateTestRoots, *stateTestHarness)
+	}{
+		{name: "resolution", pipelineID: "missing", wantError: `pipeline "missing"`},
+		{name: "pipeline", pipelineID: "sample", wantError: "synthetic extraction failure", wantValidation: "failed", configureFailure: func(_ *testing.T, _ stateTestRoots, h *stateTestHarness) {
+			h.extractErr = errors.New("synthetic extraction failure")
+		}},
+		{name: "output", pipelineID: "sample", wantError: "create output parent", wantOutputs: 1, wantValidation: "approved", configureFailure: func(t *testing.T, roots stateTestRoots, _ *stateTestHarness) {
+			if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil {
+				t.Fatal(err)
+			}
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			roots := newStateTestRoots(t)
+			harness := newStateTestHarness()
+			if tc.configureFailure != nil {
+				tc.configureFailure(t, roots, harness)
+			}
+			opts := harness.options()
+			var stdout, stderr bytes.Buffer
+			args := []string{"run", tc.pipelineID, "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}
+			code := RunWithOptions(args, &stdout, &stderr, opts)
+			if code != 1 || !strings.Contains(stderr.String(), tc.wantError) {
+				t.Fatalf("code=%d stderr=%q", code, stderr.String())
+			}
+			bundlePath := onlyChildDir(t, roots.debug)
+			runID := filepath.Base(bundlePath)
+			report := readStateTestRunReport(t, bundlePath)
+			if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != tc.wantValidation {
+				t.Fatalf("failure report = %#v", report)
+			}
+			errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
+			if err != nil || !strings.Contains(string(errorLog), tc.wantError) {
+				t.Fatalf("error log = %q, %v", errorLog, err)
+			}
+		})
+	}
+}
+
+func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
+	roots := newStateTestRoots(t)
+	harness := newStateTestHarness()
+	harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}}
+	harness.extractErr = errors.New("synthetic partial pipeline failure")
+
+	result := runStateTest(t, roots, harness.options(), true, true, "bypass")
+	if result.code != 1 {
+		t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
+	}
+	bundlePath := onlyChildDir(t, roots.debug)
+	report := readStateTestRunReport(t, bundlePath)
+	if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningCount != 1 || report.ValidationStatus != "failed" {
+		t.Fatalf("partial failure report = %#v", report)
+	}
+
+	var manifest artifacts.RunManifest
+	readStateTestSummaryJSON(t, bundlePath, "run-manifest.json", &manifest)
+	if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
+		t.Fatalf("partial manifest = %#v", manifest)
+	}
+	var warnings []contracts.Warning
+	readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings)
+	if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" {
+		t.Fatalf("partial warnings = %#v", warnings)
+	}
+	var events []pipeline.CheckpointEvent
+	readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
+	if len(events) == 0 || events[0].Stage != "source" {
+		t.Fatalf("partial checkpoint events = %#v, want retained source decision", events)
+	}
+	var chunkPlan artifacts.ChunkPlanSummary
+	readStateTestSummaryJSON(t, bundlePath, "chunk-plan.json", &chunkPlan)
+	if chunkPlan.Mode != "bypass" || chunkPlan.ValidationStatus == "not_run" {
+		t.Fatalf("partial chunk plan = %#v", chunkPlan)
+	}
+}
+
+func TestRunTerminalPersistenceFailuresDoNotRecurseOrHidePrimaryError(t *testing.T) {
+	for _, tc := range []struct {
+		name          string
+		reportErr     error
+		errorLogErr   error
+		wantSecondary string
+	}{
+		{name: "run report", reportErr: errors.New("injected run report failure"), wantSecondary: "injected run report failure"},
+		{name: "error log", errorLogErr: errors.New("injected error log failure"), wantSecondary: "injected error log failure"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			roots := newStateTestRoots(t)
+			harness := newStateTestHarness()
+			harness.extractErr = errors.New("primary pipeline failure")
+			opts := harness.options()
+			var terminal *recordingTerminalWriter
+			opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
+				terminal = &recordingTerminalWriter{delegate: delegate, reportErr: tc.reportErr, errorLogErr: tc.errorLogErr}
+				return terminal
+			}
+
+			result := runStateTest(t, roots, opts, true, false, "bypass")
+			if result.code != 1 {
+				t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
+			}
+			if terminal == nil {
+				t.Fatal("terminal writer was not constructed")
+			}
+			if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 {
+				t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls)
+			}
+			primaryIndex := strings.Index(result.stderr, "primary pipeline failure")
+			secondaryIndex := strings.Index(result.stderr, tc.wantSecondary)
+			debugIndex := strings.Index(result.stderr, "debug=")
+			if primaryIndex < 0 || secondaryIndex <= primaryIndex || debugIndex <= secondaryIndex {
+				t.Fatalf("stderr order = %q", result.stderr)
+			}
+		})
+	}
+}
+
+func TestRunReportFailureOnSuccessIsTerminalizedWithoutRetry(t *testing.T) {
+	roots := newStateTestRoots(t)
+	opts := newStateTestHarness().options()
+	var terminal *recordingTerminalWriter
+	opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
+		terminal = &recordingTerminalWriter{delegate: delegate, reportErr: errors.New("injected success report failure")}
+		return terminal
+	}
+
+	result := runStateTest(t, roots, opts, true, false, "bypass")
+	if result.code != 1 || !strings.Contains(result.stderr, "write debug run report") || !strings.Contains(result.stderr, "injected success report failure") {
+		t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
+	}
+	if terminal == nil {
+		t.Fatal("terminal writer was not constructed")
+	}
+	if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 {
+		t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls)
+	}
+	if result.stdout != "" {
+		t.Fatalf("stdout=%q, want no success message", result.stdout)
+	}
+	bundlePath := onlyChildDir(t, roots.debug)
+	errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
+	if err != nil || !strings.Contains(string(errorLog), "injected success report failure") {
+		t.Fatalf("error log = %q, %v", errorLog, err)
+	}
+}
+
+func TestRunWithoutDebugDoesNotUseTerminalSummaryWriter(t *testing.T) {
+	roots := newStateTestRoots(t)
+	harness := newStateTestHarness()
+	harness.extractErr = errors.New("non-debug pipeline failure")
+	opts := harness.options()
+	factoryCalls := 0
+	opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
+		factoryCalls++
+		return delegate
+	}
+
+	result := runStateTest(t, roots, opts, false, false, "bypass")
+	if result.code != 1 || !strings.Contains(result.stderr, "non-debug pipeline failure") {
+		t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
+	}
+	if factoryCalls != 0 {
+		t.Fatalf("terminal summary factory calls = %d, want 0", factoryCalls)
+	}
+	assertAbsent(t, roots.debug)
 }

 func TestRunRefusesExistingOutputDirectoryWithoutChangingIt(t *testing.T) {
@@ -524,6 +707,24 @@ func readAllFiles(t *testing.T, root string) string {
 	return content.String()
 }

+func readStateTestRunReport(t *testing.T, bundlePath string) debugbundle.RunReport {
+	t.Helper()
+	var report debugbundle.RunReport
+	readStateTestSummaryJSON(t, bundlePath, "run-report.json", &report)
+	return report
+}
+
+func readStateTestSummaryJSON(t *testing.T, bundlePath, name string, target any) {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join(bundlePath, "summary", name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := json.Unmarshal(data, target); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func assertRestrictedTree(t *testing.T, root string) {
 	t.Helper()
 	if runtime.GOOS == "windows" {
@@ -588,6 +789,7 @@ type stateTestHarness struct {
 	chunkCalls, extractCalls int
 	runIDCalls               uint64
 	extractErr               error
+	chunkWarnings            []contracts.Warning
 }

 func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
@@ -639,7 +841,7 @@ func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (c
 	c.harness.mu.Lock()
 	c.harness.chunkCalls++
 	c.harness.mu.Unlock()
-	return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}}, nil
+	return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil
 }

 const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
@@ -705,3 +907,25 @@ type failingDebugRecorder struct{}
 func (failingDebugRecorder) Enabled() bool                   { return true }
 func (failingDebugRecorder) WriteJSON(string, any) error     { return errors.New("trace unavailable") }
 func (failingDebugRecorder) WriteBytes(string, []byte) error { return errors.New("trace unavailable") }
+
+type recordingTerminalWriter struct {
+	delegate                   DebugTerminalWriter
+	reportErr, errorLogErr     error
+	reportCalls, errorLogCalls int
+}
+
+func (w *recordingTerminalWriter) WriteRunReport(report debugbundle.RunReport) error {
+	w.reportCalls++
+	if w.reportErr != nil {
+		return w.reportErr
+	}
+	return w.delegate.WriteRunReport(report)
+}
+
+func (w *recordingTerminalWriter) WriteError(message string) error {
+	w.errorLogCalls++
+	if w.errorLogErr != nil {
+		return w.errorLogErr
+	}
+	return w.delegate.WriteError(message)
+}