package cli import ( "bytes" "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "runtime" "strings" "sync" "testing" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) const stateTestDigest = "sha256:e511d8906649b78eb639b11215fa57a9652a1a64f4aefa3ed68320dbda46f439" func TestRunStateSurfaceMatrix(t *testing.T) { for _, debug := range []bool{false, true} { for _, resume := range []bool{false, true} { for _, mode := range []string{"auto", "bypass", "refresh"} { name := fmt.Sprintf("debug=%t/resume=%t/cache=%s", debug, resume, mode) t.Run(name, func(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() opts := harness.options() var storeRoots []string opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) { storeRoots = append(storeRoots, root) return chunkplan.NewFilesystemStore(root) } result := runStateTest(t, roots, opts, debug, resume, mode) if result.code != 0 { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } assertStateTestOutput(t, roots.output) if mode == "bypass" { assertAbsent(t, roots.plans) if len(storeRoots) != 0 { t.Fatalf("chunk plan store roots = %v, want none", storeRoots) } } else { assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json")) if len(storeRoots) != 1 || storeRoots[0] != roots.plans { t.Fatalf("chunk plan store roots = %v, want [%q]", storeRoots, roots.plans) } } assertAnyFile(t, roots.checkpoints) assertRestrictedTree(t, roots.checkpoints) if debug { bundle := onlyChildDir(t, roots.debug) assertFile(t, filepath.Join(bundle, "summary", "invocation.json")) assertAnyFile(t, filepath.Join(bundle, "trace")) assertRestrictedTree(t, roots.debug) } else { assertAbsent(t, roots.debug) } }) } } } } func TestRunKeepsStateRootsIndependentAndReusesSelectedCheckpointRoot(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() first := runStateTest(t, roots, harness.options(), true, false, "auto") if first.code != 0 { t.Fatalf("first run code=%d stderr=%q", first.code, first.stderr) } planPath := filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json") initialPlan, err := os.ReadFile(planPath) if err != nil { t.Fatal(err) } firstBundle := onlyChildDir(t, roots.debug) second := runStateTest(t, roots, harness.options(), false, false, "auto") if second.code != 0 { t.Fatalf("second run code=%d stderr=%q", second.code, second.stderr) } if harness.chunkCalls != 1 { t.Fatalf("chunk calls after debug toggle = %d, want 1", harness.chunkCalls) } if harness.extractCalls != 2 { t.Fatalf("extract calls after two recording-only runs = %d, want 2", harness.extractCalls) } if got, err := os.ReadFile(planPath); err != nil || !bytes.Equal(got, initialPlan) { t.Fatalf("chunk plan changed after debug toggle: %v", err) } if _, err := os.Stat(firstBundle); err != nil { t.Fatalf("initial debug bundle was removed: %v", err) } checkpointRoot := roots.checkpoints extractCallsBeforeResume := harness.extractCalls seed := runStateTest(t, roots, harness.options(), false, true, "auto") if seed.code != 0 { t.Fatalf("checkpoint seed code=%d stderr=%q", seed.code, seed.stderr) } if harness.extractCalls != extractCallsBeforeResume { t.Fatalf("extract calls after reusing recording-only checkpoint = %d, want %d", harness.extractCalls, extractCallsBeforeResume) } extractCalls := harness.extractCalls checkpointFiles := readTree(t, checkpointRoot) reused := runStateTest(t, roots, harness.options(), false, true, "auto") if reused.code != 0 { t.Fatalf("checkpoint reuse code=%d stderr=%q", reused.code, reused.stderr) } if harness.extractCalls != extractCalls { t.Fatalf("extract calls after checkpoint reuse = %d, want %d", harness.extractCalls, extractCalls) } if got := readTree(t, checkpointRoot); !sameFiles(got, checkpointFiles) { t.Fatal("reused checkpoint was rewritten") } } func TestRunRecomputesOnlyAfterExplicitChunkPlanRemoval(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() first := runStateTest(t, roots, harness.options(), true, false, "auto") if first.code != 0 { t.Fatalf("first run code=%d stderr=%q", first.code, first.stderr) } firstOutput := onlyChildDir(t, roots.output) firstBundle := onlyChildDir(t, roots.debug) entry := filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:")) if err := os.RemoveAll(entry); err != nil { t.Fatal(err) } second := runStateTest(t, roots, harness.options(), false, false, "auto") if second.code != 0 { t.Fatalf("second run code=%d stderr=%q", second.code, second.stderr) } if harness.chunkCalls != 2 { t.Fatalf("chunk calls = %d, want 2 after removing exact cache entry", harness.chunkCalls) } assertFile(t, filepath.Join(firstOutput, "result.json")) assertFile(t, filepath.Join(firstBundle, "summary", "run-report.json")) } func TestRunRetainsDebugBundlesAcrossFailures(t *testing.T) { t.Run("configuration failure precedes allocation", func(t *testing.T) { root := filepath.Join(t.TempDir(), "debug") var stdout, stderr bytes.Buffer code := RunWithOptions([]string{"run", "sample", "--config", filepath.Join(t.TempDir(), "missing.yml"), "--input", "missing", "--debug", "--debug-dir", root}, &stdout, &stderr, newStateTestHarness().options()) if code != 1 || !strings.Contains(stderr.String(), "config file") { t.Fatalf("code=%d stderr=%q", code, stderr.String()) } assertAbsent(t, root) }) for _, failure := range []struct { name string expected string setup func(*testing.T, stateTestRoots, *stateTestHarness) Options }{ {"resolution", "pipeline \"missing\"", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { return h.options() }}, {"pipeline", "synthetic extraction failure", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { h.extractErr = errors.New("synthetic extraction failure") return h.options() }}, {"output", "create output parent", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil { t.Fatal(err) } return h.options() }}, {"summary", "write debug invocation metadata", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { opts := h.options() opts.DebugRecorderFactory = func(traceRoot string) (pipeline.DebugRecorder, error) { if err := os.RemoveAll(filepath.Join(filepath.Dir(traceRoot), "summary")); err != nil { return nil, err } if err := os.WriteFile(filepath.Join(filepath.Dir(traceRoot), "summary"), []byte("blocked"), 0o600); err != nil { return nil, err } return frameworkdebug.NewFilesystemRecorder(traceRoot) } return opts }}, {"trace", "trace unavailable", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { opts := h.options() opts.DebugRecorderFactory = func(string) (pipeline.DebugRecorder, error) { return failingDebugRecorder{}, nil } return opts }}, } { t.Run(failure.name, func(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() opts := failure.setup(t, roots, harness) failureStderr := "" if failure.name == "resolution" { var stdout, stderr bytes.Buffer code := RunWithOptions([]string{"run", "missing", "--config", roots.config, "--input", roots.input, "--debug"}, &stdout, &stderr, opts) if code != 1 { t.Fatalf("code=%d stderr=%q", code, stderr.String()) } failureStderr = stderr.String() } else { result := runStateTest(t, roots, opts, true, false, "bypass") if result.code != 1 { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } failureStderr = result.stderr } if !strings.Contains(failureStderr, failure.expected) || !strings.Contains(failureStderr, "debug=") { t.Fatalf("stderr=%q, want %q and debug path", failureStderr, failure.expected) } bundle := onlyChildDir(t, roots.debug) if !strings.Contains(readAllFiles(t, bundle), "synthetic") && failure.name == "pipeline" { t.Fatal("pipeline failure was not retained in debug bundle") } }) } } func TestRunDebugArtifactsRedactSecretsButRetainApplicationData(t *testing.T) { roots := newStateTestRoots(t) t.Setenv("STATE_TEST_UNRELATED_ENV", "HOST_ONLY_SENTINEL") if err := os.WriteFile(filepath.Join(filepath.Dir(roots.input), "unrelated.txt"), []byte("HOST_ONLY_FILE_SENTINEL"), 0o600); err != nil { t.Fatal(err) } harness := newStateTestHarness() result := runStateTest(t, roots, harness.options(), true, false, "bypass") if result.code != 0 { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } bundle := onlyChildDir(t, roots.debug) summary := readAllFiles(t, filepath.Join(bundle, "summary")) trace := readAllFiles(t, filepath.Join(bundle, "trace")) for _, forbidden := range []string{"sk-secretvalue", "Bearer secretvalue", "HOST_ONLY_SENTINEL", "HOST_ONLY_FILE_SENTINEL"} { if strings.Contains(summary, forbidden) || strings.Contains(trace, forbidden) { t.Fatalf("debug bundle contains %q", forbidden) } } if strings.Contains(summary, "application content") { t.Fatal("summary contains raw application input") } if !strings.Contains(trace, "application content") { t.Fatal("trace does not retain expected application input") } } func TestRunRedactsSensitiveModuleOptionsFromConfigAndPipelineSummaries(t *testing.T) { roots := newStateTestRoots(t) data, err := os.ReadFile(roots.config) if err != nil { t.Fatal(err) } configText := replaceRequiredOnce(t, string(data), " input: test/input\n", ` input: module: test/input options: api_key: CONFIG_SUMMARY_SECRET_SENTINEL safe: SAFE_OPTION_SENTINEL nested: - - password: PIPELINE_SUMMARY_SECRET_SENTINEL neighbor: SAFE_NESTED_OPTION_SENTINEL `) if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil { t.Fatal(err) } result := runStateTest(t, roots, newStateTestHarness().options(), true, false, "bypass") if result.code != 0 { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } summaryRoot := filepath.Join(onlyChildDir(t, roots.debug), "summary") for _, name := range []string{"effective-config.json", "resolved-pipeline.json"} { contents, err := os.ReadFile(filepath.Join(summaryRoot, name)) if err != nil { t.Fatal(err) } text := string(contents) for _, secret := range []string{"CONFIG_SUMMARY_SECRET_SENTINEL", "PIPELINE_SUMMARY_SECRET_SENTINEL"} { if strings.Contains(text, secret) { t.Fatalf("%s contains %q: %s", name, secret, text) } } for _, retained := range []string{"[REDACTED]", "SAFE_OPTION_SENTINEL", "SAFE_NESTED_OPTION_SENTINEL"} { if !strings.Contains(text, retained) { t.Fatalf("%s does not contain %q: %s", name, retained, text) } } } } func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() opts := harness.options() const runID = "run-1000000000-11111111111111111111111111111111" opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil } result := runStateTest(t, roots, opts, true, false, "bypass") if result.code != 0 { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } outputPath := filepath.Join(roots.output, runID) debugPath := filepath.Join(roots.debug, runID) assertFile(t, filepath.Join(outputPath, "result.json")) assertFile(t, filepath.Join(debugPath, "summary", "run-manifest.json")) if !strings.Contains(result.stdout, "output="+outputPath) || !strings.Contains(result.stdout, "debug="+debugPath) { t.Fatalf("stdout=%q, want shared run identity", result.stdout) } data, err := os.ReadFile(filepath.Join(debugPath, "summary", "run-manifest.json")) if err != nil { t.Fatal(err) } var manifest artifacts.RunManifest if err := json.Unmarshal(data, &manifest); err != nil { t.Fatal(err) } if manifest.RunID != runID { t.Fatalf("manifest run ID = %q, want %q", manifest.RunID, runID) } wantStartedAt := time.Unix(1, 0).UTC() if manifest.StartedAt == nil || !manifest.StartedAt.Equal(wantStartedAt) { t.Fatalf("manifest started at = %v, want %v", manifest.StartedAt, wantStartedAt) } var invocation debugbundle.Invocation readStateTestSummaryJSON(t, debugPath, "invocation.json", &invocation) if invocation.RunID != runID || !invocation.StartedAt.Equal(wantStartedAt) { t.Fatalf("debug invocation identity = %#v, want run %q at %v", invocation, runID, wantStartedAt) } report := readStateTestRunReport(t, debugPath) if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" { t.Fatalf("success report = %#v", report) } if !strings.Contains(result.stdout, "outputs=1 rejected=0") { t.Fatalf("stdout=%q, want report counts", result.stdout) } } func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *testing.T) { for _, tc := range []struct { name string pipelineID string wantError string wantOutputs int wantValidation string configureFailure func(*testing.T, stateTestRoots, *stateTestHarness) }{ {name: "resolution", pipelineID: "missing", wantError: `pipeline "missing"`}, {name: "pipeline", pipelineID: "sample", wantError: "synthetic extraction failure", wantValidation: "failed", configureFailure: func(_ *testing.T, _ stateTestRoots, h *stateTestHarness) { h.extractErr = errors.New("synthetic extraction failure") }}, {name: "output", pipelineID: "sample", wantError: "create output parent", wantOutputs: 1, wantValidation: "approved", configureFailure: func(t *testing.T, roots stateTestRoots, _ *stateTestHarness) { if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil { t.Fatal(err) } }}, } { t.Run(tc.name, func(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() if tc.configureFailure != nil { tc.configureFailure(t, roots, harness) } opts := harness.options() var stdout, stderr bytes.Buffer args := []string{"run", tc.pipelineID, "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"} code := RunWithOptions(args, &stdout, &stderr, opts) if code != 1 || !strings.Contains(stderr.String(), tc.wantError) { t.Fatalf("code=%d stderr=%q", code, stderr.String()) } bundlePath := onlyChildDir(t, roots.debug) runID := filepath.Base(bundlePath) report := readStateTestRunReport(t, bundlePath) if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != tc.wantValidation { t.Fatalf("failure report = %#v", report) } errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log")) if err != nil || !strings.Contains(string(errorLog), tc.wantError) { t.Fatalf("error log = %q, %v", errorLog, err) } }) } } func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}} harness.extractErr = errors.New("synthetic partial pipeline failure") result := runStateTest(t, roots, harness.options(), true, true, "bypass") if result.code != 1 { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } bundlePath := onlyChildDir(t, roots.debug) report := readStateTestRunReport(t, bundlePath) if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningCount != 1 || report.ValidationStatus != "failed" { t.Fatalf("partial failure report = %#v", report) } var manifest artifacts.RunManifest readStateTestSummaryJSON(t, bundlePath, "run-manifest.json", &manifest) if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" { t.Fatalf("partial manifest = %#v", manifest) } var warnings []contracts.Warning readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings) if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" { t.Fatalf("partial warnings = %#v", warnings) } var events []pipeline.CheckpointEvent readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events) if len(events) == 0 || events[0].Stage != "source" { t.Fatalf("partial checkpoint events = %#v, want retained source decision", events) } var chunkPlan artifacts.ChunkPlanSummary readStateTestSummaryJSON(t, bundlePath, "chunk-plan.json", &chunkPlan) if chunkPlan.Mode != "bypass" || chunkPlan.ValidationStatus == "not_run" { t.Fatalf("partial chunk plan = %#v", chunkPlan) } } func TestRunTerminalPersistenceFailuresDoNotRecurseOrHidePrimaryError(t *testing.T) { for _, tc := range []struct { name string reportErr error errorLogErr error wantSecondary string }{ {name: "run report", reportErr: errors.New("injected run report failure"), wantSecondary: "injected run report failure"}, {name: "error log", errorLogErr: errors.New("injected error log failure"), wantSecondary: "injected error log failure"}, } { t.Run(tc.name, func(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() harness.extractErr = errors.New("primary pipeline failure") opts := harness.options() var terminal *recordingTerminalWriter opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter { terminal = &recordingTerminalWriter{delegate: delegate, reportErr: tc.reportErr, errorLogErr: tc.errorLogErr} return terminal } result := runStateTest(t, roots, opts, true, false, "bypass") if result.code != 1 { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } if terminal == nil { t.Fatal("terminal writer was not constructed") } if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 { t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls) } primaryIndex := strings.Index(result.stderr, "primary pipeline failure") secondaryIndex := strings.Index(result.stderr, tc.wantSecondary) debugIndex := strings.Index(result.stderr, "debug=") if primaryIndex < 0 || secondaryIndex <= primaryIndex || debugIndex <= secondaryIndex { t.Fatalf("stderr order = %q", result.stderr) } }) } } func TestRunReportFailureOnSuccessIsTerminalizedWithoutRetry(t *testing.T) { roots := newStateTestRoots(t) opts := newStateTestHarness().options() var terminal *recordingTerminalWriter opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter { terminal = &recordingTerminalWriter{delegate: delegate, reportErr: errors.New("injected success report failure")} return terminal } result := runStateTest(t, roots, opts, true, false, "bypass") if result.code != 1 || !strings.Contains(result.stderr, "write debug run report") || !strings.Contains(result.stderr, "injected success report failure") { t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) } if terminal == nil { t.Fatal("terminal writer was not constructed") } if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 { t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls) } if result.stdout != "" { t.Fatalf("stdout=%q, want no success message", result.stdout) } bundlePath := onlyChildDir(t, roots.debug) errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log")) if err != nil || !strings.Contains(string(errorLog), "injected success report failure") { t.Fatalf("error log = %q, %v", errorLog, err) } } func TestRunWithoutDebugDoesNotUseTerminalSummaryWriter(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() harness.extractErr = errors.New("non-debug pipeline failure") opts := harness.options() factoryCalls := 0 opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter { factoryCalls++ return delegate } result := runStateTest(t, roots, opts, false, false, "bypass") if result.code != 1 || !strings.Contains(result.stderr, "non-debug pipeline failure") { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } if factoryCalls != 0 { t.Fatalf("terminal summary factory calls = %d, want 0", factoryCalls) } assertAbsent(t, roots.debug) } func TestRunRefusesExistingOutputDirectoryWithoutChangingIt(t *testing.T) { roots := newStateTestRoots(t) const runID = "run-1000000000-22222222222222222222222222222222" runPath := filepath.Join(roots.output, runID) if err := os.MkdirAll(filepath.Join(runPath, "nested"), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(runPath, "sentinel"), []byte("existing output"), 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(runPath, "nested", "data"), []byte("preserve me"), 0o644); err != nil { t.Fatal(err) } before := readTree(t, runPath) opts := newStateTestHarness().options() opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil } result := runStateTest(t, roots, opts, true, false, "bypass") if result.code != 1 || !strings.Contains(result.stderr, "output run directory") || !strings.Contains(result.stderr, "already exists") { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } if after := readTree(t, runPath); !sameFiles(after, before) { t.Fatalf("existing output changed: before=%v after=%v", before, after) } bundlePath := filepath.Join(roots.debug, runID) report := readStateTestRunReport(t, bundlePath) if report.Succeeded || report.RunID != runID || report.OutputPath != runPath || report.DebugPath != bundlePath || report.OutputCount != 1 || report.ValidationStatus != "approved" { t.Fatalf("output collision report = %#v", report) } errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log")) if err != nil || !strings.Contains(string(errorLog), "already exists") { t.Fatalf("output collision error log = %q, %v", errorLog, err) } } func TestRepeatedRunIdentityCannotOverwriteFirstOutput(t *testing.T) { roots := newStateTestRoots(t) const runID = "run-1000000000-33333333333333333333333333333333" harness := newStateTestHarness() opts := harness.options() opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil } first := runStateTest(t, roots, opts, false, false, "bypass") if first.code != 0 { t.Fatalf("first code=%d stderr=%q", first.code, first.stderr) } runPath := filepath.Join(roots.output, runID) before := readTree(t, runPath) second := runStateTest(t, roots, opts, false, false, "bypass") if second.code != 1 || !strings.Contains(second.stderr, "already exists") { t.Fatalf("second code=%d stderr=%q", second.code, second.stderr) } if after := readTree(t, runPath); !sameFiles(after, before) { t.Fatalf("first output changed: before=%v after=%v", before, after) } } func TestRunRefusesExistingDebugBundleWithoutChangingIt(t *testing.T) { roots := newStateTestRoots(t) const runID = "run-1000000000-44444444444444444444444444444444" bundlePath := filepath.Join(roots.debug, runID) if err := os.MkdirAll(bundlePath, 0o700); err != nil { t.Fatal(err) } sentinelPath := filepath.Join(bundlePath, "sentinel") if err := os.WriteFile(sentinelPath, []byte("existing debug"), 0o600); err != nil { t.Fatal(err) } opts := newStateTestHarness().options() opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil } result := runStateTest(t, roots, opts, true, false, "bypass") if result.code != 1 || !strings.Contains(result.stderr, "debug bundle") || !strings.Contains(result.stderr, "already exists") { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } if got, err := os.ReadFile(sentinelPath); err != nil || string(got) != "existing debug" { t.Fatalf("sentinel = %q, %v", got, err) } assertAbsent(t, roots.output) } func TestRunIDGenerationFailurePrecedesDebugAllocation(t *testing.T) { roots := newStateTestRoots(t) opts := newStateTestHarness().options() opts.RunIDGenerator = func(time.Time) (string, error) { return "", errors.New("random source unavailable") } result := runStateTest(t, roots, opts, true, false, "bypass") if result.code != 1 || !strings.Contains(result.stderr, "generate run ID: random source unavailable") { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } assertAbsent(t, roots.debug) assertAbsent(t, roots.output) } func TestRunRejectsUnsafeGeneratedIdentityBeforePathUse(t *testing.T) { roots := newStateTestRoots(t) opts := newStateTestHarness().options() opts.RunIDGenerator = func(time.Time) (string, error) { return "../outside", nil } result := runStateTest(t, roots, opts, true, false, "bypass") if result.code != 1 || !strings.Contains(result.stderr, "invalid generated run ID") || !strings.Contains(result.stderr, "one safe path component") { t.Fatalf("code=%d stderr=%q", result.code, result.stderr) } assertAbsent(t, roots.debug) assertAbsent(t, roots.output) } type stateTestRoots struct{ config, input, output, plans, checkpoints, debug string } func newStateTestRoots(t *testing.T) stateTestRoots { t.Helper() base := t.TempDir() roots := stateTestRoots{input: filepath.Join(base, "input.txt"), output: filepath.Join(base, "output"), plans: filepath.Join(base, "plans"), checkpoints: filepath.Join(base, "checkpoints"), debug: filepath.Join(base, "debug")} if err := os.WriteFile(roots.input, []byte("application content Bearer secretvalue sk-secretvalue"), 0o600); err != nil { t.Fatal(err) } roots.config = filepath.Join(base, "config.yml") config := fmt.Sprintf("version: 4\noutput:\n directory: %q\ncache:\n chunk_plans:\n directory: %q\n mode: auto\n checkpoints:\n enabled: true\n directory: %q\ndebug:\n directory: %q\npipelines:\n sample:\n input: test/input\n chunk: test/chunk\n artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n output: test/output\n", roots.output, roots.plans, roots.checkpoints, roots.debug) if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil { t.Fatal(err) } return roots } type stateTestResult struct { code int stdout, stderr string } func runStateTest(t *testing.T, roots stateTestRoots, opts Options, debug, resume bool, mode string) stateTestResult { t.Helper() args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", mode} if debug { args = append(args, "--debug") } if resume { args = append(args, "--resume") } var stdout, stderr bytes.Buffer return stateTestResult{RunWithOptions(args, &stdout, &stderr, opts), stdout.String(), stderr.String()} } func assertStateTestOutput(t *testing.T, root string) { t.Helper() output := onlyChildDir(t, root) data, err := os.ReadFile(filepath.Join(output, "result.json")) if err != nil || string(data) != "{\"ok\":true}\n" { t.Fatalf("output = %q, %v", data, err) } } func onlyChildDir(t *testing.T, root string) string { t.Helper() entries, err := os.ReadDir(root) if err != nil { t.Fatal(err) } var dirs []string for _, entry := range entries { if entry.IsDir() { dirs = append(dirs, filepath.Join(root, entry.Name())) } } if len(dirs) != 1 { t.Fatalf("directories in %q = %v, want one", root, dirs) } return dirs[0] } func assertFile(t *testing.T, path string) { t.Helper() if info, err := os.Stat(path); err != nil || info.IsDir() { t.Fatalf("file %q: %v", path, err) } } func assertAbsent(t *testing.T, path string) { t.Helper() if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatalf("%q exists or stat failed: %v", path, err) } } func assertAnyFile(t *testing.T, root string) { t.Helper() if text := readAllFiles(t, root); text == "" { t.Fatalf("no files under %q", root) } } func readAllFiles(t *testing.T, root string) string { t.Helper() var content strings.Builder if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { if err != nil { return err } if entry.IsDir() { return nil } data, err := os.ReadFile(path) if err != nil { return err } content.Write(data) return nil }); err != nil { t.Fatal(err) } return content.String() } func readStateTestRunReport(t *testing.T, bundlePath string) debugbundle.RunReport { t.Helper() var report debugbundle.RunReport readStateTestSummaryJSON(t, bundlePath, "run-report.json", &report) return report } func readStateTestSummaryJSON(t *testing.T, bundlePath, name string, target any) { t.Helper() data, err := os.ReadFile(filepath.Join(bundlePath, "summary", name)) if err != nil { t.Fatal(err) } if err := json.Unmarshal(data, target); err != nil { t.Fatal(err) } } func assertRestrictedTree(t *testing.T, root string) { t.Helper() if runtime.GOOS == "windows" { return } if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { return err } want := os.FileMode(0o600) if info.IsDir() { want = 0o700 } if info.Mode().Perm() != want { return fmt.Errorf("%s has mode %o, want %o", path, info.Mode().Perm(), want) } return nil }); err != nil { t.Fatal(err) } } func readTree(t *testing.T, root string) map[string][]byte { t.Helper() files := map[string][]byte{} if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { if err != nil { return err } if entry.IsDir() { return nil } data, err := os.ReadFile(path) if err != nil { return err } relative, err := filepath.Rel(root, path) if err != nil { return err } files[relative] = data return nil }); err != nil { t.Fatal(err) } return files } func sameFiles(left, right map[string][]byte) bool { if len(left) != len(right) { return false } for path, data := range left { if !bytes.Equal(data, right[path]) { return false } } return true } type stateTestHarness struct { mu sync.Mutex chunkCalls, extractCalls int runIDCalls uint64 extractErr error chunkWarnings []contracts.Warning moduleProfiles []string sessionIDs []string outputWarnings []contracts.Warning includeWarnings bool } func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} } func (h *stateTestHarness) options() Options { registries := pipeline.Registries{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()} if err := pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, stateTestCodec{}); err != nil { panic(err) } if err := registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil { panic(err) } if err := registries.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "cache-reference"}}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil { panic(err) } if err := pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "test/extract", Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{h}, nil }); err != nil { panic(err) } if err := pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "test/merge", Stage: pipeline.StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{harness: h}, nil }); err != nil { panic(err) } if err := pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "test/normalize", Stage: pipeline.StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{harness: h}, nil }); err != nil { panic(err) } if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, ExecutionClass: contracts.ExecutionClassDeterministic, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) { return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil }); err != nil { panic(err) } return Options{Catalog: catalogFromRegistries(registries), Registries: registries, LookupEnv: emptyLookup, Now: func() time.Time { return time.Unix(1, 0) }, RunIDGenerator: func(startedAt time.Time) (string, error) { h.mu.Lock() defer h.mu.Unlock() h.runIDCalls++ return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil }, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { return nil, nil, nil }} } type stateTestInput struct{} func (stateTestInput) Key() string { return "test/input" } func (stateTestInput) Parse(_ context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) { doc := &source.SourceDocument{ID: "source", Kind: "text", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "text", Text: string(req.Raw), Ref: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}}} digest, err := source.DigestDocument(doc) if err != nil { return nil, err } doc.Digest = digest return doc, nil } type stateTestChunker struct{ harness *stateTestHarness } func (stateTestChunker) Key() string { return "test/chunk" } func (stateTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { c.harness.mu.Lock() c.harness.moduleProfiles = append(c.harness.moduleProfiles, req.LLMProfile) c.harness.sessionIDs = append(c.harness.sessionIDs, req.SessionID) c.harness.mu.Unlock() c.harness.mu.Lock() c.harness.chunkCalls++ c.harness.mu.Unlock() return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil } const stateTestArtifactKind contracts.ArtifactKind = "test/artifact" type stateTestArtifact struct { Value string `json:"value"` } type stateTestCodec struct{} func (stateTestCodec) Kind() contracts.ArtifactKind { return stateTestArtifactKind } func (stateTestCodec) Schema() contracts.ArtifactSchema { return contracts.ArtifactSchema{ID: "test.artifact", Name: "test_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)} } func (stateTestCodec) MediaType() string { return "application/json" } func (stateTestCodec) EncodeCandidate(v stateTestArtifact) ([]byte, error) { return []byte(`{"value":"ok"}`), nil } func (stateTestCodec) Encode(v stateTestArtifact) ([]byte, error) { return []byte(`{"value":"ok"}`), nil } func (stateTestCodec) Decode([]byte) (stateTestArtifact, error) { return stateTestArtifact{Value: "ok"}, nil } type stateTestExtractor struct{ harness *stateTestHarness } func (stateTestExtractor) Key() string { return "test/extract" } func (stateTestExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (e stateTestExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[stateTestArtifact], error) { e.harness.mu.Lock() defer e.harness.mu.Unlock() e.harness.extractCalls++ e.harness.moduleProfiles = append(e.harness.moduleProfiles, req.LLMProfile) e.harness.sessionIDs = append(e.harness.sessionIDs, req.SessionID) if e.harness.extractErr != nil { return contracts.TypedExtractionResult[stateTestArtifact]{}, e.harness.extractErr } return contracts.TypedExtractionResult[stateTestArtifact]{Value: stateTestArtifact{Value: "ok"}}, nil } type stateTestMerger struct{ harness *stateTestHarness } func (stateTestMerger) Key() string { return "test/merge" } func (m stateTestMerger) Merge(_ context.Context, req contracts.TypedMergeRequest[stateTestArtifact]) (contracts.TypedMergeResult[stateTestArtifact], error) { m.harness.mu.Lock() m.harness.moduleProfiles = append(m.harness.moduleProfiles, req.LLMProfile) m.harness.sessionIDs = append(m.harness.sessionIDs, req.SessionID) m.harness.mu.Unlock() return contracts.TypedMergeResult[stateTestArtifact]{Value: req.ExtractOutputs[0].Value}, nil } type stateTestNormalizer struct{ harness *stateTestHarness } func (stateTestNormalizer) Key() string { return "test/normalize" } func (stateTestNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[stateTestArtifact]) (contracts.TypedNormalizeResult[stateTestArtifact], error) { n.harness.mu.Lock() n.harness.moduleProfiles = append(n.harness.moduleProfiles, req.LLMProfile) n.harness.sessionIDs = append(n.harness.sessionIDs, req.SessionID) n.harness.mu.Unlock() return contracts.TypedNormalizeResult[stateTestArtifact]{Value: req.MergeOutput.Value}, nil } type stateTestOutput struct { harness *stateTestHarness includeWarnings bool } func (o stateTestOutput) Key() string { return "test/output" } func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { o.harness.mu.Lock() o.harness.outputWarnings = append([]contracts.Warning(nil), req.Warnings...) o.harness.mu.Unlock() data := []byte("{\"ok\":true}\n") if o.includeWarnings && len(req.Warnings) > 0 { data = []byte(fmt.Sprintf("{\"ok\":true,\"warnings\":%q}\n", req.Warnings[0].ReasonCode)) } return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil } type failingDebugRecorder struct{} func (failingDebugRecorder) Enabled() bool { return true } func (failingDebugRecorder) WriteJSON(string, any) error { return errors.New("trace unavailable") } func (failingDebugRecorder) WriteBytes(string, []byte) error { return errors.New("trace unavailable") } type recordingTerminalWriter struct { delegate DebugTerminalWriter reportErr, errorLogErr error reportCalls, errorLogCalls int } func (w *recordingTerminalWriter) WriteRunReport(report debugbundle.RunReport) error { w.reportCalls++ if w.reportErr != nil { return w.reportErr } return w.delegate.WriteRunReport(report) } func (w *recordingTerminalWriter) WriteError(message string) error { w.errorLogCalls++ if w.errorLogErr != nil { return w.errorLogErr } return w.delegate.WriteError(message) }