diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 2686c2c..d35236f 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -709,6 +709,17 @@ reported at finalization without erasing a completed transcript. ## Stage 11 — Make Debug Persistence Explicit and Independent +**Completed 2026-09-24.** Debug directory creation and all current debug writer +paths are gated by `diagnostics.debug`; failures are reported safely to stderr +without aborting processing. The retained directory uses the supplied run ID, +old retention modes are gone, raw and normalized snapshots are redacted, and the +candidate ledger includes proposals not applied after filtering or interruption. +Tests cover explicit candidate capture, retained debug evidence, secret redaction, +and writer behavior. `go test ./internal/core/redaction +./internal/core/diagnostics ./internal/framework/runner +./internal/framework/processreport` and `git diff --check` passed. CLI package +compilation is deferred to Stage 12's replacement of the old report lifecycle. + **Work:** Refactor `core/diagnostics` run-directory creation and all CLI/module/ proposal/validator writer wiring for H. Only debug creates the unique retained run directory. Gate snapshots, interaction files, ledger, and full execution data diff --git a/internal/cli/run.go b/internal/cli/run.go index a26225c..20b5fb5 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -28,6 +28,7 @@ type processInvocation struct { GlossaryPath string OutputPath string ReportDir string + RunID string Config config.Config ConfigResult *ck.Result[config.Config] LLMRuntime *llm.Runtime @@ -37,46 +38,65 @@ type processInvocation struct { ConfigVersion *int } -var processRunner = func(ctx context.Context, inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) { - runDir, err := diagnostics.NewRunDirectory(inv.Config.Diagnostics.WorkDir, "always", llm.ConfiguredSecrets(&inv.Config)...) - if err != nil { - return nil, nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err) +var processRunner = func(ctx context.Context, inv processInvocation, stdout, stderr io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) { + var runDir *diagnostics.RunDirectory + if inv.Config.Diagnostics.Debug { + var err error + runDir, err = diagnostics.NewRunDirectory(inv.Config.Diagnostics.WorkDir, inv.RunID, llm.ConfiguredSecrets(&inv.Config)...) + if err != nil { + fmt.Fprintf(stderr, "audita process: debug directory: %s\n", llm.RedactError(err, inv.Config)) + } + } + debugPath := "" + if runDir != nil { + debugPath = runDir.Path() + } + debugWrite := func(label string, write func(*diagnostics.RunDirectory) error) { + if runDir == nil { + return + } + if err := write(runDir); err != nil { + fmt.Fprintf(stderr, "audita process: debug %s: %s\n", label, llm.RedactError(err, inv.Config)) + } } fail := func(phase string, err error, runOutput *runner.RunOutput) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) { - _ = runDir.WriteErrorLog(fmt.Sprintf("%s: %s", phase, llm.RedactError(err, inv.Config))) + debugWrite("error log", func(d *diagnostics.RunDirectory) error { + return d.WriteErrorLog(fmt.Sprintf("%s: %s", phase, llm.RedactError(err, inv.Config))) + }) return nil, nil, runOutput, runDir, fmt.Errorf("%s: %w", phase, err) } - if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{ - Operation: "process", - TranscriptPath: inv.TranscriptPath, - GlossaryPath: inv.GlossaryPath, - OutputPath: inv.OutputPath, - ConfigPath: inv.ConfigPath, - ConfigSource: inv.ConfigSource, - Overlay: inv.Overlay, - ConfigVersion: inv.ConfigVersion, - TranscriptDescription: inv.Config.Context.Description, - Modules: append([]string(nil), inv.Config.Pipeline.Modules...), - }); err != nil { - _ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err)) - } - if inv.ConfigResult != nil { + debugWrite("invocation", func(d *diagnostics.RunDirectory) error { + return d.WriteInvocationMetadata(diagnostics.InvocationMetadata{ + Operation: "process", + TranscriptPath: inv.TranscriptPath, + GlossaryPath: inv.GlossaryPath, + OutputPath: inv.OutputPath, + ConfigPath: inv.ConfigPath, + ConfigSource: inv.ConfigSource, + Overlay: inv.Overlay, + ConfigVersion: inv.ConfigVersion, + TranscriptDescription: inv.Config.Context.Description, + Modules: append([]string(nil), inv.Config.Pipeline.Modules...), + }) + }) + if runDir != nil && inv.ConfigResult != nil { var effective bytes.Buffer if err := inv.ConfigResult.WriteEffective(&effective); err == nil { - err = runDir.WriteEffectiveConfig(effective.Bytes()) - } - if err != nil { - _ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err)) + debugWrite("effective settings", func(d *diagnostics.RunDirectory) error { return d.WriteEffectiveConfig(effective.Bytes()) }) } if origins, err := llm.ExplanationYAML(inv.ConfigResult); err == nil { - _ = runDir.WriteYAMLArtifact(diagnostics.ArtifactConfigOrigins, origins) + debugWrite("configuration origins", func(d *diagnostics.RunDirectory) error { + return d.WriteYAMLArtifact(diagnostics.ArtifactConfigOrigins, origins) + }) } } - if inv.LLMRuntime != nil { + if runDir != nil && inv.LLMRuntime != nil { if profiles, err := inv.LLMRuntime.SelectedProfilesYAML(); err == nil { - _ = runDir.WriteYAMLArtifact(diagnostics.ArtifactSelectedProfiles, profiles) + debugWrite("selected profiles", func(d *diagnostics.RunDirectory) error { + return d.WriteYAMLArtifact(diagnostics.ArtifactSelectedProfiles, profiles) + }) } } @@ -100,9 +120,9 @@ var processRunner = func(ctx context.Context, inv processInvocation, stdout io.W return fail("glossary_schema", err, nil) } - if err := runDir.WriteSourceTranscript(sourceTranscript, transcriptBytes); err != nil { - _ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err)) - } + debugWrite("source transcript", func(d *diagnostics.RunDirectory) error { + return d.WriteSourceTranscript(sourceTranscript, transcriptBytes) + }) canonical := sourceToCanonicalTranscript(sourceTranscript) normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{ @@ -114,12 +134,8 @@ var processRunner = func(ctx context.Context, inv processInvocation, stdout io.W normalizedTranscript, normSummary := normalizer.Normalize(canonical) - if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil { - _ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err)) - } - if err := runDir.WriteNormalizationSummary(normSummary); err != nil { - _ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err)) - } + debugWrite("normalized transcript", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizedTranscript(normalizedTranscript) }) + debugWrite("normalization summary", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizationSummary(normSummary) }) // Compute chunks after normalization chunker := chunking.NewChunker(chunking.ChunkingConfig{ @@ -141,14 +157,12 @@ var processRunner = func(ctx context.Context, inv processInvocation, stdout io.W chunkSummary := chunking.ComputeSummary(sections, chunkConfig) chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig) - if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil { - _ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err)) - } + debugWrite("chunking summary", func(d *diagnostics.RunDirectory) error { return d.WriteChunkingSummary(&chunkDetailedSummary) }) workingTranscript := normalizedTranscript var runOutput *runner.RunOutput observations := llm.NewObservationCollector() - moduleFactory := modules.NewFactory(modules.Dependencies{Config: &inv.Config, Glossary: glossary, DiagnosticsDir: runDir.Path()}) + moduleFactory := modules.NewFactory(modules.Dependencies{Config: &inv.Config, Glossary: glossary, DiagnosticsDir: debugPath}) if inv.LLMRuntime == nil { return fail("runner_setup", fmt.Errorf("LLM runtime is required"), nil) } @@ -173,9 +187,10 @@ var processRunner = func(ctx context.Context, inv processInvocation, stdout io.W ModuleSpecs: moduleSpecs, ProposalLLMClient: proposalLLMClient, - ProposalDiagnosticsDir: runDir.Path(), + ProposalDiagnosticsDir: debugPath, ValidationLLMClient: validationLLMClient, - ValidationDiagnosticsDir: runDir.Path(), + ValidationDiagnosticsDir: debugPath, + CaptureCandidates: runDir != nil, }) runnerResult.Observations = observations.Snapshot() runnerResult.ObservationsAvailable = true @@ -311,26 +326,24 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in ConfigVersion: &version, } - normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(ctx, inv, stdout) + normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(ctx, inv, stdout, stderr) completedAt := time.Now().UTC() if runErr != nil { if runDir != nil && runOutput != nil { - writeExecutionArtifacts(runDir, runOutput, inv.Config) + writeExecutionArtifacts(runDir, runOutput, inv.Config, stderr) - _ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{ + if debugErr := runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{ RunDirectoryPath: runDir.Path(), RunOutput: runOutput, - })) + })); debugErr != nil { + fmt.Fprintf(stderr, "audita process: debug correction ledger: %s\n", llm.RedactError(debugErr, inv.Config)) + } } errorPhase, errorMessage := extractErrorPhase(runErr) errorMessage = llm.RedactError(fmt.Errorf("%s", errorMessage), inv.Config) report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput)) - if runDir != nil { - _ = runDir.WriteReport(report) - } - fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(runErr, inv.Config)) if runDir != nil { fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path()) @@ -339,29 +352,31 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in } if runDir != nil && runOutput != nil { - writeExecutionArtifacts(runDir, runOutput, inv.Config) + writeExecutionArtifacts(runDir, runOutput, inv.Config, stderr) - _ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{ + if debugErr := runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{ RunDirectoryPath: runDir.Path(), RunOutput: runOutput, - })) + })); debugErr != nil { + fmt.Fprintf(stderr, "audita process: debug correction ledger: %s\n", llm.RedactError(debugErr, inv.Config)) + } } report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput)) - if runDir != nil { - _ = runDir.WriteReport(report) - } - return 0 } -func writeExecutionArtifacts(runDir *diagnostics.RunDirectory, output *runner.RunOutput, cfg config.Config) { +func writeExecutionArtifacts(runDir *diagnostics.RunDirectory, output *runner.RunOutput, cfg config.Config, stderr io.Writer) { if !output.ObservationsAvailable { return } - _ = runDir.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations)) - _ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, processreport.BuildUtilization(output, cfg.Backends)) + if err := runDir.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations)); err != nil { + fmt.Fprintf(stderr, "audita process: debug model execution: %s\n", llm.RedactError(err, cfg)) + } + if err := runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, processreport.BuildUtilization(output, cfg.Backends)); err != nil { + fmt.Fprintf(stderr, "audita process: debug utilization: %s\n", llm.RedactError(err, cfg)) + } } func runConfig(ctx context.Context, args []string, stdout, stderr io.Writer) int { diff --git a/internal/core/diagnostics/output_session_test.go b/internal/core/diagnostics/output_session_test.go index 949cc0f..c88462e 100644 --- a/internal/core/diagnostics/output_session_test.go +++ b/internal/core/diagnostics/output_session_test.go @@ -199,3 +199,30 @@ func TestFinalizationCollectsFailuresAndRetainsPublishedTranscript(t *testing.T) } } } + +func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) { + parent := filepath.Join(t.TempDir(), "debug") + if _, err := os.Stat(parent); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected parent: %v", err) + } + run, err := NewRunDirectory(parent, "run-123", "secret") + if err != nil { + t.Fatal(err) + } + if run.Path() != filepath.Join(parent, "run-123") { + t.Fatalf("run ID changed: %s", run.Path()) + } + if _, err := NewRunDirectory(parent, "run-123"); err == nil { + t.Fatal("duplicate run ID accepted") + } + if err := run.WriteJSONArtifact(ArtifactCorrectionLedger, map[string]string{"proposal": "secret"}); err != nil { + t.Fatal(err) + } + contents, err := os.ReadFile(filepath.Join(run.Path(), ArtifactCorrectionLedger)) + if err != nil || strings.Contains(string(contents), "secret") { + t.Fatalf("debug artifact: %s %v", contents, err) + } + if _, err := os.Stat(run.Path()); err != nil { + t.Fatalf("debug directory removed: %v", err) + } +} diff --git a/internal/core/diagnostics/retention_test.go b/internal/core/diagnostics/retention_test.go deleted file mode 100644 index 548d55e..0000000 --- a/internal/core/diagnostics/retention_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package diagnostics - -import ( - "os" - "path/filepath" - "testing" -) - -func TestShouldRetainRunDirectoryMatrix(t *testing.T) { - tests := []struct { - name string - input RetentionDecisionInput - want bool - }{ - {name: "always success keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: true}, want: true}, - {name: "always failure keeps", input: RetentionDecisionInput{RetentionMode: "always", RunSucceeded: false}, want: true}, - {name: "never success keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: true}, want: true}, - {name: "never failure keeps", input: RetentionDecisionInput{RetentionMode: "never", RunSucceeded: false}, want: true}, - {name: "auto success no skips removes", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: false}, want: false}, - {name: "auto success skips keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: true, HasSkippedCorrections: true}, want: true}, - {name: "auto failure keeps", input: RetentionDecisionInput{RetentionMode: "auto", RunSucceeded: false}, want: true}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := ShouldRetainRunDirectory(tc.input) - if got != tc.want { - t.Fatalf("unexpected retain decision: got=%v want=%v", got, tc.want) - } - }) - } -} - -func TestApplyRetentionRemovesWhenDecisionSaysRemove(t *testing.T) { - workDir := t.TempDir() - runPath := filepath.Join(workDir, "run-test") - if err := os.Mkdir(runPath, 0o755); err != nil { - t.Fatalf("mkdir run path: %v", err) - } - - runDir := &RunDirectory{path: runPath, retention: "auto"} - err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false}) - if err != nil { - t.Fatalf("ApplyRetention failed: %v", err) - } - - if _, err := os.Stat(runPath); !os.IsNotExist(err) { - t.Fatalf("expected run directory removed, stat err=%v", err) - } -} - -func TestApplyRetentionReturnsRemovalError(t *testing.T) { - parent := t.TempDir() - runPath := filepath.Join(parent, "run-test") - if err := os.Mkdir(runPath, 0o755); err != nil { - t.Fatalf("mkdir run path: %v", err) - } - - // Make parent non-writable so removing child fails. - if err := os.Chmod(parent, 0o500); err != nil { - t.Fatalf("chmod parent: %v", err) - } - t.Cleanup(func() { - _ = os.Chmod(parent, 0o700) - }) - - runDir := &RunDirectory{path: runPath, retention: "auto"} - err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasSkippedCorrections: false}) - if err == nil { - t.Fatalf("expected removal error, got nil") - } -} diff --git a/internal/core/diagnostics/run_dir.go b/internal/core/diagnostics/run_dir.go index 85c3a8c..5565d31 100644 --- a/internal/core/diagnostics/run_dir.go +++ b/internal/core/diagnostics/run_dir.go @@ -1,9 +1,11 @@ package diagnostics import ( + "encoding/json" "fmt" "os" "path/filepath" + "strings" "time" "gitea.maximumdirect.net/eric/audita/internal/core/chunking" @@ -16,42 +18,15 @@ import ( type RunDirectory struct { redactor redaction.Redactor path string - retention string createdAt time.Time } -type RetentionDecisionInput struct { - RetentionMode string - RunSucceeded bool - HasSkippedCorrections bool -} - -func ShouldRetainRunDirectory(input RetentionDecisionInput) bool { - // Failed runs are always retained. - if !input.RunSucceeded { - return true - } - - switch input.RetentionMode { - case "always": - return true - case "never": - return true - case "auto": - return input.HasSkippedCorrections - default: - // Be conservative for unknown values. - return true - } -} - // InvocationMetadata captures non-secret invocation details for diagnostics. type InvocationMetadata struct { Operation string `json:"operation"` TranscriptPath string `json:"transcript_path"` GlossaryPath string `json:"glossary_path"` OutputPath string `json:"output_path,omitempty"` - ReportJSONPath string `json:"report_json_path,omitempty"` ConfigPath string `json:"config_path,omitempty"` ConfigSource string `json:"config_source,omitempty"` Overlay string `json:"overlay,omitempty"` @@ -63,9 +38,12 @@ type InvocationMetadata struct { } // NewRunDirectory creates a new run directory under the configured work dir -func NewRunDirectory(workDir, retention string, secrets ...string) (*RunDirectory, error) { - if workDir == "" { - workDir = ".audita-runs" +func NewRunDirectory(workDir, runID string, secrets ...string) (*RunDirectory, error) { + if strings.TrimSpace(workDir) == "" { + return nil, fmt.Errorf("debug work directory is required") + } + if strings.TrimSpace(runID) == "" || filepath.Base(runID) != runID || runID == "." || runID == ".." { + return nil, fmt.Errorf("valid run ID is required") } // Create work directory if it doesn't exist @@ -73,8 +51,6 @@ func NewRunDirectory(workDir, retention string, secrets ...string) (*RunDirector return nil, fmt.Errorf("failed to create work directory %q: %w", workDir, err) } - // Create a unique run directory identifier. - runID := fmt.Sprintf("run-%d", time.Now().UTC().UnixNano()) runPath := filepath.Join(workDir, runID) if err := os.Mkdir(runPath, 0o755); err != nil { @@ -84,7 +60,6 @@ func NewRunDirectory(workDir, retention string, secrets ...string) (*RunDirector return &RunDirectory{ path: runPath, redactor: redaction.New(secrets), - retention: retention, createdAt: time.Now().UTC(), }, nil } @@ -126,9 +101,19 @@ func (r *RunDirectory) WriteEffectiveConfig(effectiveYAML []byte) error { // WriteSourceTranscript writes the source transcript artifact func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error { - // Write raw source for reference + // Preserve the source snapshot without copying unredacted input bytes. + var source any + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + if err := decoder.Decode(&source); err != nil { + return fmt.Errorf("decode source snapshot: %w", err) + } + sourceBytes, err := r.redactor.JSON(source) + if err != nil { + return fmt.Errorf("redact source snapshot: %w", err) + } sourcePath := filepath.Join(r.path, ArtifactSourceTranscript) - if err := os.WriteFile(sourcePath, raw, 0o644); err != nil { + if err := os.WriteFile(sourcePath, append(sourceBytes, '\n'), 0o644); err != nil { return fmt.Errorf("failed to write source transcript: %w", err) } @@ -149,7 +134,7 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript // WriteNormalizedTranscript writes the normalized transcript artifact func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error { normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript) - bytes, err := schema.TranscriptToJSON(transcript) + bytes, err := r.redactor.JSON(transcript) if err != nil { return fmt.Errorf("failed to serialize normalized transcript: %w", err) } @@ -218,15 +203,3 @@ func (r *RunDirectory) WriteYAMLArtifact(name string, payload []byte) error { } return nil } - -func (r *RunDirectory) ApplyRetention(input RetentionDecisionInput) error { - decision := input - if decision.RetentionMode == "" { - decision.RetentionMode = r.retention - } - - if ShouldRetainRunDirectory(decision) { - return nil - } - return os.RemoveAll(r.path) -} diff --git a/internal/core/redaction/redaction_test.go b/internal/core/redaction/redaction_test.go index c32e851..7e2eab1 100644 --- a/internal/core/redaction/redaction_test.go +++ b/internal/core/redaction/redaction_test.go @@ -10,7 +10,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/diagnostics" "gitea.maximumdirect.net/eric/audita/internal/core/redaction" - "gitea.maximumdirect.net/eric/audita/internal/core/reporting" + "gitea.maximumdirect.net/eric/audita/internal/core/schema" ) func TestStringRedactionPreservesSerializationAndDomainData(t *testing.T) { @@ -51,37 +51,26 @@ func TestStringRedactionPreservesSerializationAndDomainData(t *testing.T) { } } -func TestReportWritersRedactStringsAndFailClosed(t *testing.T) { +func TestDebugWritersRedactSourceAndNormalizedSnapshots(t *testing.T) { dir := t.TempDir() - version := 2 - report := reporting.ProcessReport{ErrorMessage: "echo:2"} - report.ReportMetadata.ConfigVersion = &version - run, err := diagnostics.NewRunDirectory(dir, "always", "2") + run, err := diagnostics.NewRunDirectory(dir, "run-test", "2") if err != nil { t.Fatal(err) } - external := filepath.Join(dir, "external.json") - if err := reporting.WriteProcessReport(external, report, "2"); err != nil { + if err := run.WriteSourceTranscript(&schema.SourceTranscript{}, []byte(`{"segments":[{"text":"echo:2","id":2}]}`)); err != nil { t.Fatal(err) } - if err := run.WriteReport(report); err != nil { + if err := run.WriteNormalizedTranscript(&schema.Transcript{Segments: []schema.Segment{{ID: 2, Text: "echo:2"}}}); err != nil { t.Fatal(err) } - for _, path := range []string{external, filepath.Join(run.Path(), diagnostics.ArtifactReport)} { + for _, path := range []string{filepath.Join(run.Path(), diagnostics.ArtifactSourceTranscript), filepath.Join(run.Path(), diagnostics.ArtifactNormalizedTranscript)} { raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } - var got reporting.ProcessReport - if err := json.Unmarshal(raw, &got); err != nil { - t.Fatal(err) + if strings.Contains(string(raw), "echo:2") || !strings.Contains(string(raw), "echo:[REDACTED]") || !strings.Contains(string(raw), `"id": 2`) { + t.Fatalf("snapshot: %s", raw) } - if got.ErrorMessage != "echo:[REDACTED]" || got.ReportMetadata.ConfigVersion == nil || *got.ReportMetadata.ConfigVersion != 2 { - t.Fatalf("report: %+v", got) - } - } - if report.ErrorMessage != "echo:2" { - t.Fatal("mutated input report") } if err := run.WriteJSONArtifact("invalid.json", map[string]any{"secret": "echo:2", "invalid": math.NaN()}); err == nil { t.Fatal("expected serialization failure") diff --git a/internal/framework/processreport/correction_ledger.go b/internal/framework/processreport/correction_ledger.go index bb038a6..cc5e433 100644 --- a/internal/framework/processreport/correction_ledger.go +++ b/internal/framework/processreport/correction_ledger.go @@ -1,6 +1,7 @@ package processreport import ( + "fmt" "path/filepath" "sort" @@ -9,10 +10,11 @@ import ( ) const ( - correctionDispositionApplied = "applied" - correctionDispositionRejected = "rejected" - correctionDispositionSkipped = "skipped" - correctionDispositionFailed = "failed" + correctionDispositionApplied = "applied" + correctionDispositionRejected = "rejected" + correctionDispositionSkipped = "skipped" + correctionDispositionFailed = "failed" + correctionDispositionInterrupted = "interrupted" ) type CorrectionLedgerInput struct { @@ -55,6 +57,8 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry } entries := make([]CorrectionLedgerEntry, 0) + seen := make(map[string]bool) + identity := func(instance string, index int) string { return instance + "\x00" + fmt.Sprint(index) } for _, module := range runOutput.ModuleResults { decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord) for _, decision := range module.ValidatorDecisions { @@ -62,6 +66,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry } for _, change := range module.AppliedChanges { + seen[identity(module.ModuleInstance, change.ProposalIndex)] = true entries = append(entries, CorrectionLedgerEntry{ RunID: runID, ModuleKey: module.ModuleKey, @@ -78,6 +83,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry }) } for _, change := range module.SkippedChanges { + seen[identity(module.ModuleInstance, change.ProposalIndex)] = true entries = append(entries, CorrectionLedgerEntry{ RunID: runID, ModuleKey: module.ModuleKey, @@ -95,6 +101,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry }) } for _, rejection := range module.ValidatorRejected { + seen[identity(module.ModuleInstance, rejection.ProposalIndex)] = true entries = append(entries, CorrectionLedgerEntry{ RunID: runID, ModuleKey: module.ModuleKey, @@ -122,6 +129,34 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry }) } } + for _, candidate := range runOutput.Candidates { + if seen[identity(candidate.ModuleInstance, candidate.ProposalIndex)] { + continue + } + var decisions []runner.ValidatorDecisionRecord + policy := "" + for _, module := range runOutput.ModuleResults { + if module.ModuleInstance != candidate.ModuleInstance { + continue + } + policy = string(module.ReplacementPolicy) + for _, decision := range module.ValidatorDecisions { + if decision.ProposalIndex == candidate.ProposalIndex { + decisions = append(decisions, decision) + } + } + break + } + entries = append(entries, CorrectionLedgerEntry{ + RunID: runID, ModuleKey: candidate.ModuleKey, ModuleInstance: candidate.ModuleInstance, + ProposalIndex: candidate.ProposalIndex, SegmentID: candidate.TargetSegmentID, + OriginalText: candidate.OriginalText, ProposedCorrectedText: candidate.CorrectedText, + ReplacementPolicy: policy, Disposition: correctionDispositionInterrupted, + DispositionReasonCode: "not_applied", DispositionMessage: "candidate did not reach application", + DeterministicValidatorResults: filterLedgerDecisions(decisions, false), + LLMValidatorResults: filterLedgerDecisions(decisions, true), + }) + } sort.SliceStable(entries, func(i, j int) bool { if entries[i].ModuleInstance != entries[j].ModuleInstance { diff --git a/internal/framework/processreport/correction_ledger_test.go b/internal/framework/processreport/correction_ledger_test.go index 288dd0f..386006d 100644 --- a/internal/framework/processreport/correction_ledger_test.go +++ b/internal/framework/processreport/correction_ledger_test.go @@ -126,3 +126,25 @@ func TestHasSkippedCorrectionsIncludesApplicationSkipsAndValidatorRejections(t * t.Fatal("validator rejections should count as skipped corrections") } } + +func TestCorrectionLedgerIncludesUnappliedCandidateAfterFatalValidation(t *testing.T) { + output := &runner.RunOutput{ + Candidates: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"}, ProposalMetadata: proposals.ProposalMetadata{ModuleKey: "grammar", ModuleInstance: "grammar", ProposalIndex: 2}}}, + ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar", Status: runner.ModuleStatusFailed, + ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "spoken_form_plausibility", ProposalIndex: 2, Approved: false, ReasonCode: "transport_failed"}}}}, + } + ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output}) + if len(ledger) != 2 { + t.Fatalf("missing candidate: %+v", ledger) + } + for _, entry := range ledger { + if entry.ProposalIndex != 2 { + continue + } + if entry.Disposition != correctionDispositionInterrupted || entry.OriginalText != "old" || len(entry.LLMValidatorResults) != 1 { + t.Fatalf("candidate evidence lost: %+v", entry) + } + return + } + t.Fatal("candidate missing") +} diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 2f8c7e2..21881e2 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -94,20 +94,22 @@ type RunInput struct { ProposalDiagnosticsDir string ValidationLLMClient contracts.StructuredLLMClient ValidationDiagnosticsDir string + CaptureCandidates bool timings *validatorTimingCollector } // RunOutput is the deterministic runner output. type RunOutput struct { - FinalTranscript *schema.Transcript `json:"-"` - ModuleResults []ModuleResult `json:"module_results"` - Sections []contracts.SectionMetadata `json:"sections"` - WorkUnits []WorkUnitOutcome `json:"work_units"` - Observations llm.ObservationSnapshot `json:"-"` - ObservationsAvailable bool `json:"-"` - LLMSummary llm.ExecutionSummary `json:"-"` - RunWallTimeMS int64 `json:"-"` - ValidatorTimings []ValidatorTiming `json:"-"` + FinalTranscript *schema.Transcript `json:"-"` + ModuleResults []ModuleResult `json:"module_results"` + Sections []contracts.SectionMetadata `json:"sections"` + WorkUnits []WorkUnitOutcome `json:"work_units"` + Candidates []proposals.EnrichedCorrectionProposal `json:"-"` + Observations llm.ObservationSnapshot `json:"-"` + ObservationsAvailable bool `json:"-"` + LLMSummary llm.ExecutionSummary `json:"-"` + RunWallTimeMS int64 `json:"-"` + ValidatorTimings []ValidatorTiming `json:"-"` } func New(factory ModuleFactory) *Runner { @@ -136,6 +138,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) { } inventory := make([]contracts.SectionMetadata, len(sections)) workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections)) + var candidates []proposals.EnrichedCorrectionProposal for i, section := range sections { inventory[i] = contracts.SectionMetadataFromSection(section) } @@ -151,7 +154,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) { snapshot := func() RunOutput { return RunOutput{ FinalTranscript: transcriptFromSections(sections), ModuleResults: results, - Sections: inventory, WorkUnits: workUnits, + Sections: inventory, WorkUnits: workUnits, Candidates: candidates, } } @@ -185,6 +188,9 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) { ValidationDiagnosticsDir: input.ValidationDiagnosticsDir, Timings: input.timings, }) + if input.CaptureCandidates { + candidates = append(candidates, pipelineResult.Candidates...) + } if pipelineErr != nil { failed := ModuleResult{ ModuleKey: spec.ModuleKey, @@ -321,6 +327,7 @@ type sectionValidationResult struct { type modulePipelineResult struct { ProposalCount int + Candidates []proposals.EnrichedCorrectionProposal Approved []proposals.EnrichedCorrectionProposal Operations []outcomes.Operation WorkUnits []WorkUnitOutcome @@ -473,6 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } nextProposalIndex += len(sectionEnriched) out.ProposalCount += len(sectionEnriched) + out.Candidates = append(out.Candidates, sectionEnriched...) for operationIndex := range out.WorkUnits[nextSectionToProcess].Operations { operation := &out.WorkUnits[nextSectionToProcess].Operations[operationIndex] if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 72e97c1..22b5186 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -128,6 +128,31 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) { } } +func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) { + module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, + validators: []contracts.Validator{validators.ProposalShapeValidator{}}, + proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good", Confidence: 1}}, nil + }} + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}) + input := RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "bad"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}} + without, err := r.Run(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(without.Candidates) != 0 { + t.Fatalf("debug-off candidate capture: %+v", without.Candidates) + } + input.CaptureCandidates = true + with, err := r.Run(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(with.Candidates) != 1 || with.Candidates[0].ProposalIndex != 0 || len(with.ModuleResults[0].ValidatorRejected) != 1 { + t.Fatalf("filtered candidate missing from debug evidence: %+v", with) + } +} + func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { return m.proposeResult(req) }