eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Write terminal debug reports for failed runs

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-18 14:31:46 +00:00
parent 2111e01142
commit e4ec521bed
6 changed files with 370 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
docs/internal/state.md
View File

@@ -43,6 +43,12 @@ boundaries redact sensitive metadata and credential-shaped bytes while allowing
application-owned trace material. Debug data is never a checkpoint source or
cache input.
After allocation, one CLI-owned state value accumulates the known report paths,
pipeline outcome counts, and validation status. A single guarded terminalization
operation writes the success report, or makes one attempt each to write the
failure report and error log. Terminal persistence failures are reported
separately and never replace the command's primary error.
## Tests To Inspect
- `internal/cli/state_surfaces_test.go`: debug allocation and configuration

16
docs/operations.md
View File

@@ -140,16 +140,20 @@ supported Unix systems.
Notarius never automatically deletes a requested bundle. If allocation
succeeds, its path is reported on success and failure. A requested summary or
trace write failure makes the command fail, preserving whatever bundle data was
already written for inspection.
already written for inspection. Every allocated bundle makes one best-effort
attempt to record a terminal `run-report.json`.
## Failures And Warnings
Failures before debug allocation are reported on stderr without a bundle.
Failures after allocation report the bundle path on stderr and write `error.log`
when that summary write succeeds. An output-write failure leaves the allocated
bundle in place. A successful run with warnings exits `0`, reports a warning
count on stderr, and records warnings in durable output and any requested debug
summary.
Failures after allocation report the bundle path on stderr and make independent
attempts to write a failure `run-report.json` and `error.log`. The report retains
the paths and pipeline outcome fields known at the failure point. If either
terminal write fails, the original command error remains first on stderr,
followed by the persistence error and bundle path. An output-write failure
leaves the allocated bundle in place. A successful run with warnings exits `0`,
reports a warning count on stderr, and records warnings in durable output and
any requested debug summary.
## Cleanup

2
docs/roadmap/implementation.md
View File

@@ -277,7 +277,7 @@ all repository checks pass.
## Stage 3: Write terminal run reports for failures
**Status:** Not started
**Status:** Complete
### Objective

78
internal/cli/run.go
View File

@@ -43,6 +43,7 @@ type Options struct {
UserCacheDir func() (string, error)
ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
DebugRecorderFactory func(string) (pipeline.DebugRecorder, error)
DebugTerminalFactory func(*debugbundle.SummaryWriter) DebugTerminalWriter
}
type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)
@@ -104,6 +105,9 @@ func normalizeOptions(opts Options) (Options, error) {
if opts.DebugRecorderFactory == nil {
opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder
}
if opts.DebugTerminalFactory == nil {
opts.DebugTerminalFactory = func(writer *debugbundle.SummaryWriter) DebugTerminalWriter { return writer }
}
if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) {
components, err := newProductionComponents()
if err != nil {
@@ -226,7 +230,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: invalid generated run ID: %v\n", err)
return 1
}
runOutputDir := filepath.Join(cfg.Output.Directory, runID)
commandState := newPipelineCommandState(runID, pipelineID, runOutputDir)
var summary *debugbundle.SummaryWriter
var terminalWriter DebugTerminalWriter
debugPath := ""
debugRecorder := pipeline.NoopDebugRecorder()
if *debug {
@@ -236,9 +243,14 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return 1
}
debugPath, summary = bundle.Path(), bundle.Summary()
commandState.setDebugPath(debugPath)
terminalWriter = opts.DebugTerminalFactory(summary)
if terminalWriter == nil {
terminalWriter = summary
}
debugRecorder, err = opts.DebugRecorderFactory(bundle.TraceRoot())
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("create debug recorder: %w", err), true)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create debug recorder: %w", err))
}
debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder)
}
@@ -255,16 +267,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
StartedAt: startedAt,
}
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug invocation metadata: %w", err), false)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
}
catalog, err := effectiveCatalog(opts)
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, err, true)
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, err, true)
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
effective, err := cfg.Resolve(config.ResolveInput{
PipelineID: pipelineID,
@@ -275,43 +287,43 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
ReferenceUnbinds: referenceUnbinds,
})
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, err, true)
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, profileIDs); err != nil {
return failPipelineCommand(stderr, summary, debugPath, err, true)
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
workingDir, err := os.Getwd()
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("resolve working directory: %w", err), true)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
}
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
ConfigPath: loadedConfigPath,
WorkingDir: workingDir,
})
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, err, true)
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
effective.ResolvedPipeline = materialized
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug invocation metadata: %w", err), false)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
}
if err := writeSummary(summary, func() error { return summary.WriteRedactedEffectiveConfig(effective) }); err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug effective config: %w", err), false)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug effective config: %w", err))
}
if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug resolved pipeline: %w", err), false)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved pipeline: %w", err))
}
if err := writeSummary(summary, func() error {
return summary.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline))
}); err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug resolved references: %w", err), false)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved references: %w", err))
}
registries, err := effectiveRegistries(opts)
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, err, true)
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
ctx := context.Background()
@@ -321,24 +333,24 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
}
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err), true)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("prepare pipeline %q: %w", pipelineID, err), true)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))
}
rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err), true)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
}
chunkPlans, err := chunkPlanStoreForRun(effective.Config.Cache.ChunkPlans, opts)
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, err, true)
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
if err != nil {
return failPipelineCommand(stderr, summary, debugPath, err, true)
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
output, err := pipeline.New().Run(ctx, pipeline.RunInput{
@@ -358,26 +370,25 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
Debug: debugRecorder,
ExtractWorkers: cfg.Concurrency.StageWorkers["extract"],
})
commandState.observeOutput(output)
if err != nil {
primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
if output.Manifest.PipelineID != "" {
if summaryErr := writePartialSummary(summary, output); summaryErr != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("run pipeline %q: %w; write debug summary: %v", pipelineID, err, summaryErr), false)
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
}
}
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("run pipeline %q: %w", pipelineID, err), true)
return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
}
runOutputDir := filepath.Join(effective.Config.Output.Directory, runID)
if err := writePartialSummary(summary, output); err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug summary: %w", err), false)
return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
}
if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil {
return failPipelineCommand(stderr, summary, debugPath, err, true)
return failPipelineCommand(stderr, commandState, terminalWriter, err)
}
if err := writeSummary(summary, func() error {
return summary.WriteRunReport(debugbundle.RunReport{RunID: runID, PipelineID: effective.PipelineID, OutputPath: runOutputDir, DebugPath: debugPath, Succeeded: true, OutputCount: len(output.NormalizeOutputs), RejectedCount: len(output.Rejected), WarningCount: len(output.Warnings), ValidationStatus: output.Manifest.ValidationStatus})
}); err != nil {
return failPipelineCommand(stderr, summary, debugPath, fmt.Errorf("write debug run report: %w", err), false)
if primaryErr, persistenceErr := commandState.terminalize(terminalWriter, nil); primaryErr != nil {
return writePipelineCommandFailure(stderr, commandState, primaryErr, persistenceErr)
}
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
@@ -390,19 +401,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return 0
}
func failPipelineCommand(stderr io.Writer, summary *debugbundle.SummaryWriter, debugPath string, err error, recordError bool) int {
fmt.Fprintf(stderr, "notarius: %v\n", err)
if recordError && summary != nil {
if summaryErr := summary.WriteError(err.Error()); summaryErr != nil {
fmt.Fprintf(stderr, "notarius: write debug error log: %v\n", summaryErr)
}
}
if debugPath != "" {
fmt.Fprintf(stderr, "notarius: debug=%s\n", debugPath)
}
return 1
}
func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
if summary == nil {
return nil

90
internal/cli/run_terminal.go Normal file
View File

@@ -0,0 +1,90 @@
package cli
import (
"errors"
"fmt"
"io"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
type DebugTerminalWriter interface {
WriteRunReport(debugbundle.RunReport) error
WriteError(string) error
}
type pipelineCommandState struct {
report debugbundle.RunReport
terminalized bool
}
func newPipelineCommandState(runID, pipelineID, outputPath string) *pipelineCommandState {
return &pipelineCommandState{report: debugbundle.RunReport{
RunID: runID,
PipelineID: pipelineID,
OutputPath: outputPath,
}}
}
func (s *pipelineCommandState) setDebugPath(debugPath string) {
if s != nil {
s.report.DebugPath = debugPath
}
}
func (s *pipelineCommandState) observeOutput(output pipeline.RunOutput) {
if s == nil {
return
}
s.report.OutputCount = len(output.NormalizeOutputs)
s.report.RejectedCount = len(output.Rejected)
s.report.WarningCount = len(output.Warnings)
s.report.ValidationStatus = output.Manifest.ValidationStatus
}
func (s *pipelineCommandState) terminalize(writer DebugTerminalWriter, primaryErr error) (error, error) {
if s == nil || s.terminalized {
return primaryErr, nil
}
s.terminalized = true
if writer == nil {
return primaryErr, nil
}
report := s.report
report.Succeeded = primaryErr == nil
reportErr := writer.WriteRunReport(report)
if reportErr != nil {
reportErr = fmt.Errorf("write debug run report: %w", reportErr)
if primaryErr == nil {
primaryErr = reportErr
reportErr = nil
}
}
var errorLogErr error
if primaryErr != nil {
if err := writer.WriteError(primaryErr.Error()); err != nil {
errorLogErr = fmt.Errorf("write debug error log: %w", err)
}
}
return primaryErr, errors.Join(reportErr, errorLogErr)
}
func failPipelineCommand(stderr io.Writer, state *pipelineCommandState, writer DebugTerminalWriter, primaryErr error, persistenceErrs ...error) int {
primaryErr, terminalErr := state.terminalize(writer, primaryErr)
persistenceErrs = append(persistenceErrs, terminalErr)
return writePipelineCommandFailure(stderr, state, primaryErr, errors.Join(persistenceErrs...))
}
func writePipelineCommandFailure(stderr io.Writer, state *pipelineCommandState, primaryErr, persistenceErr error) int {
fmt.Fprintf(stderr, "notarius: %v\n", primaryErr)
if persistenceErr != nil {
fmt.Fprintf(stderr, "notarius: %v\n", persistenceErr)
}
if state != nil && state.report.DebugPath != "" {
fmt.Fprintf(stderr, "notarius: debug=%s\n", state.report.DebugPath)
}
return 1
}

226
internal/cli/state_hardening_test.go
View File

@@ -16,6 +16,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -321,6 +322,188 @@ func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) {
if manifest.RunID != runID {
t.Fatalf("manifest run ID = %q, want %q", manifest.RunID, runID)
}
report := readStateTestRunReport(t, debugPath)
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" {
t.Fatalf("success report = %#v", report)
}
if !strings.Contains(result.stdout, "outputs=1 rejected=0") {
t.Fatalf("stdout=%q, want report counts", result.stdout)
}
}
func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *testing.T) {
for _, tc := range []struct {
name string
pipelineID string
wantError string
wantOutputs int
wantValidation string
configureFailure func(*testing.T, stateTestRoots, *stateTestHarness)
}{
{name: "resolution", pipelineID: "missing", wantError: `pipeline "missing"`},
{name: "pipeline", pipelineID: "sample", wantError: "synthetic extraction failure", wantValidation: "failed", configureFailure: func(_ *testing.T, _ stateTestRoots, h *stateTestHarness) {
h.extractErr = errors.New("synthetic extraction failure")
}},
{name: "output", pipelineID: "sample", wantError: "create output parent", wantOutputs: 1, wantValidation: "approved", configureFailure: func(t *testing.T, roots stateTestRoots, _ *stateTestHarness) {
if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil {
t.Fatal(err)
}
}},
} {
t.Run(tc.name, func(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
if tc.configureFailure != nil {
tc.configureFailure(t, roots, harness)
}
opts := harness.options()
var stdout, stderr bytes.Buffer
args := []string{"run", tc.pipelineID, "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}
code := RunWithOptions(args, &stdout, &stderr, opts)
if code != 1 || !strings.Contains(stderr.String(), tc.wantError) {
t.Fatalf("code=%d stderr=%q", code, stderr.String())
}
bundlePath := onlyChildDir(t, roots.debug)
runID := filepath.Base(bundlePath)
report := readStateTestRunReport(t, bundlePath)
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != tc.wantValidation {
t.Fatalf("failure report = %#v", report)
}
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
if err != nil || !strings.Contains(string(errorLog), tc.wantError) {
t.Fatalf("error log = %q, %v", errorLog, err)
}
})
}
}
func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}}
harness.extractErr = errors.New("synthetic partial pipeline failure")
result := runStateTest(t, roots, harness.options(), true, true, "bypass")
if result.code != 1 {
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
}
bundlePath := onlyChildDir(t, roots.debug)
report := readStateTestRunReport(t, bundlePath)
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningCount != 1 || report.ValidationStatus != "failed" {
t.Fatalf("partial failure report = %#v", report)
}
var manifest artifacts.RunManifest
readStateTestSummaryJSON(t, bundlePath, "run-manifest.json", &manifest)
if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
t.Fatalf("partial manifest = %#v", manifest)
}
var warnings []contracts.Warning
readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings)
if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" {
t.Fatalf("partial warnings = %#v", warnings)
}
var events []pipeline.CheckpointEvent
readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
if len(events) == 0 || events[0].Stage != "source" {
t.Fatalf("partial checkpoint events = %#v, want retained source decision", events)
}
var chunkPlan artifacts.ChunkPlanSummary
readStateTestSummaryJSON(t, bundlePath, "chunk-plan.json", &chunkPlan)
if chunkPlan.Mode != "bypass" || chunkPlan.ValidationStatus == "not_run" {
t.Fatalf("partial chunk plan = %#v", chunkPlan)
}
}
func TestRunTerminalPersistenceFailuresDoNotRecurseOrHidePrimaryError(t *testing.T) {
for _, tc := range []struct {
name string
reportErr error
errorLogErr error
wantSecondary string
}{
{name: "run report", reportErr: errors.New("injected run report failure"), wantSecondary: "injected run report failure"},
{name: "error log", errorLogErr: errors.New("injected error log failure"), wantSecondary: "injected error log failure"},
} {
t.Run(tc.name, func(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.extractErr = errors.New("primary pipeline failure")
opts := harness.options()
var terminal *recordingTerminalWriter
opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
terminal = &recordingTerminalWriter{delegate: delegate, reportErr: tc.reportErr, errorLogErr: tc.errorLogErr}
return terminal
}
result := runStateTest(t, roots, opts, true, false, "bypass")
if result.code != 1 {
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
}
if terminal == nil {
t.Fatal("terminal writer was not constructed")
}
if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 {
t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls)
}
primaryIndex := strings.Index(result.stderr, "primary pipeline failure")
secondaryIndex := strings.Index(result.stderr, tc.wantSecondary)
debugIndex := strings.Index(result.stderr, "debug=")
if primaryIndex < 0 || secondaryIndex <= primaryIndex || debugIndex <= secondaryIndex {
t.Fatalf("stderr order = %q", result.stderr)
}
})
}
}
func TestRunReportFailureOnSuccessIsTerminalizedWithoutRetry(t *testing.T) {
roots := newStateTestRoots(t)
opts := newStateTestHarness().options()
var terminal *recordingTerminalWriter
opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
terminal = &recordingTerminalWriter{delegate: delegate, reportErr: errors.New("injected success report failure")}
return terminal
}
result := runStateTest(t, roots, opts, true, false, "bypass")
if result.code != 1 || !strings.Contains(result.stderr, "write debug run report") || !strings.Contains(result.stderr, "injected success report failure") {
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
}
if terminal == nil {
t.Fatal("terminal writer was not constructed")
}
if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 {
t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls)
}
if result.stdout != "" {
t.Fatalf("stdout=%q, want no success message", result.stdout)
}
bundlePath := onlyChildDir(t, roots.debug)
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
if err != nil || !strings.Contains(string(errorLog), "injected success report failure") {
t.Fatalf("error log = %q, %v", errorLog, err)
}
}
func TestRunWithoutDebugDoesNotUseTerminalSummaryWriter(t *testing.T) {
roots := newStateTestRoots(t)
harness := newStateTestHarness()
harness.extractErr = errors.New("non-debug pipeline failure")
opts := harness.options()
factoryCalls := 0
opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
factoryCalls++
return delegate
}
result := runStateTest(t, roots, opts, false, false, "bypass")
if result.code != 1 || !strings.Contains(result.stderr, "non-debug pipeline failure") {
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
}
if factoryCalls != 0 {
t.Fatalf("terminal summary factory calls = %d, want 0", factoryCalls)
}
assertAbsent(t, roots.debug)
}
func TestRunRefusesExistingOutputDirectoryWithoutChangingIt(t *testing.T) {
@@ -524,6 +707,24 @@ func readAllFiles(t *testing.T, root string) string {
return content.String()
}
func readStateTestRunReport(t *testing.T, bundlePath string) debugbundle.RunReport {
t.Helper()
var report debugbundle.RunReport
readStateTestSummaryJSON(t, bundlePath, "run-report.json", &report)
return report
}
func readStateTestSummaryJSON(t *testing.T, bundlePath, name string, target any) {
t.Helper()
data, err := os.ReadFile(filepath.Join(bundlePath, "summary", name))
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(data, target); err != nil {
t.Fatal(err)
}
}
func assertRestrictedTree(t *testing.T, root string) {
t.Helper()
if runtime.GOOS == "windows" {
@@ -588,6 +789,7 @@ type stateTestHarness struct {
chunkCalls, extractCalls int
runIDCalls uint64
extractErr error
chunkWarnings []contracts.Warning
}
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
@@ -639,7 +841,7 @@ func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (c
c.harness.mu.Lock()
c.harness.chunkCalls++
c.harness.mu.Unlock()
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}}, nil
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil
}
const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
@@ -705,3 +907,25 @@ type failingDebugRecorder struct{}
func (failingDebugRecorder) Enabled() bool { return true }
func (failingDebugRecorder) WriteJSON(string, any) error { return errors.New("trace unavailable") }
func (failingDebugRecorder) WriteBytes(string, []byte) error { return errors.New("trace unavailable") }
type recordingTerminalWriter struct {
delegate DebugTerminalWriter
reportErr, errorLogErr error
reportCalls, errorLogCalls int
}
func (w *recordingTerminalWriter) WriteRunReport(report debugbundle.RunReport) error {
w.reportCalls++
if w.reportErr != nil {
return w.reportErr
}
return w.delegate.WriteRunReport(report)
}
func (w *recordingTerminalWriter) WriteError(message string) error {
w.errorLogCalls++
if w.errorLogErr != nil {
return w.errorLogErr
}
return w.delegate.WriteError(message)
}