Make debug evidence opt-in and retain all candidates

internal/cli/run.go

@@ -28,6 +28,7 @@ type processInvocation struct {
 	GlossaryPath   string
 	OutputPath     string
 	ReportDir      string
+	RunID          string
 	Config         config.Config
 	ConfigResult   *ck.Result[config.Config]
 	LLMRuntime     *llm.Runtime
@@ -37,46 +38,65 @@ type processInvocation struct {
 	ConfigVersion  *int
 }
 
-var processRunner = func(ctx context.Context, inv processInvocation, stdout io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
-	runDir, err := diagnostics.NewRunDirectory(inv.Config.Diagnostics.WorkDir, "always", llm.ConfiguredSecrets(&inv.Config)...)
-	if err != nil {
-		return nil, nil, nil, nil, fmt.Errorf("run_dir_creation: %w", err)
+var processRunner = func(ctx context.Context, inv processInvocation, stdout, stderr io.Writer) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
+	var runDir *diagnostics.RunDirectory
+	if inv.Config.Diagnostics.Debug {
+		var err error
+		runDir, err = diagnostics.NewRunDirectory(inv.Config.Diagnostics.WorkDir, inv.RunID, llm.ConfiguredSecrets(&inv.Config)...)
+		if err != nil {
+			fmt.Fprintf(stderr, "audita process: debug directory: %s\n", llm.RedactError(err, inv.Config))
+		}
+	}
+	debugPath := ""
+	if runDir != nil {
+		debugPath = runDir.Path()
+	}
+	debugWrite := func(label string, write func(*diagnostics.RunDirectory) error) {
+		if runDir == nil {
+			return
+		}
+		if err := write(runDir); err != nil {
+			fmt.Fprintf(stderr, "audita process: debug %s: %s\n", label, llm.RedactError(err, inv.Config))
+		}
 	}
 
 	fail := func(phase string, err error, runOutput *runner.RunOutput) (*normalization.NormalizationSummary, *chunking.Summary, *runner.RunOutput, *diagnostics.RunDirectory, error) {
-		_ = runDir.WriteErrorLog(fmt.Sprintf("%s: %s", phase, llm.RedactError(err, inv.Config)))
+		debugWrite("error log", func(d *diagnostics.RunDirectory) error {
+			return d.WriteErrorLog(fmt.Sprintf("%s: %s", phase, llm.RedactError(err, inv.Config)))
+		})
 		return nil, nil, runOutput, runDir, fmt.Errorf("%s: %w", phase, err)
 	}
 
-	if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
-		Operation:             "process",
-		TranscriptPath:        inv.TranscriptPath,
-		GlossaryPath:          inv.GlossaryPath,
-		OutputPath:            inv.OutputPath,
-		ConfigPath:            inv.ConfigPath,
-		ConfigSource:          inv.ConfigSource,
-		Overlay:               inv.Overlay,
-		ConfigVersion:         inv.ConfigVersion,
-		TranscriptDescription: inv.Config.Context.Description,
-		Modules:               append([]string(nil), inv.Config.Pipeline.Modules...),
-	}); err != nil {
-		_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
-	}
-	if inv.ConfigResult != nil {
+	debugWrite("invocation", func(d *diagnostics.RunDirectory) error {
+		return d.WriteInvocationMetadata(diagnostics.InvocationMetadata{
+			Operation:             "process",
+			TranscriptPath:        inv.TranscriptPath,
+			GlossaryPath:          inv.GlossaryPath,
+			OutputPath:            inv.OutputPath,
+			ConfigPath:            inv.ConfigPath,
+			ConfigSource:          inv.ConfigSource,
+			Overlay:               inv.Overlay,
+			ConfigVersion:         inv.ConfigVersion,
+			TranscriptDescription: inv.Config.Context.Description,
+			Modules:               append([]string(nil), inv.Config.Pipeline.Modules...),
+		})
+	})
+	if runDir != nil && inv.ConfigResult != nil {
 		var effective bytes.Buffer
 		if err := inv.ConfigResult.WriteEffective(&effective); err == nil {
-			err = runDir.WriteEffectiveConfig(effective.Bytes())
-		}
-		if err != nil {
-			_ = runDir.WriteErrorLog(fmt.Sprintf("effective_config: %v", err))
+			debugWrite("effective settings", func(d *diagnostics.RunDirectory) error { return d.WriteEffectiveConfig(effective.Bytes()) })
 		}
 		if origins, err := llm.ExplanationYAML(inv.ConfigResult); err == nil {
-			_ = runDir.WriteYAMLArtifact(diagnostics.ArtifactConfigOrigins, origins)
+			debugWrite("configuration origins", func(d *diagnostics.RunDirectory) error {
+				return d.WriteYAMLArtifact(diagnostics.ArtifactConfigOrigins, origins)
+			})
 		}
 	}
-	if inv.LLMRuntime != nil {
+	if runDir != nil && inv.LLMRuntime != nil {
 		if profiles, err := inv.LLMRuntime.SelectedProfilesYAML(); err == nil {
-			_ = runDir.WriteYAMLArtifact(diagnostics.ArtifactSelectedProfiles, profiles)
+			debugWrite("selected profiles", func(d *diagnostics.RunDirectory) error {
+				return d.WriteYAMLArtifact(diagnostics.ArtifactSelectedProfiles, profiles)
+			})
 		}
 	}
 
@@ -100,9 +120,9 @@ var processRunner = func(ctx context.Context, inv processInvocation, stdout io.W
 		return fail("glossary_schema", err, nil)
 	}
 
-	if err := runDir.WriteSourceTranscript(sourceTranscript, transcriptBytes); err != nil {
-		_ = runDir.WriteErrorLog(fmt.Sprintf("source_artifact: %v", err))
-	}
+	debugWrite("source transcript", func(d *diagnostics.RunDirectory) error {
+		return d.WriteSourceTranscript(sourceTranscript, transcriptBytes)
+	})
 
 	canonical := sourceToCanonicalTranscript(sourceTranscript)
 	normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
@@ -114,12 +134,8 @@ var processRunner = func(ctx context.Context, inv processInvocation, stdout io.W
 
 	normalizedTranscript, normSummary := normalizer.Normalize(canonical)
 
-	if err := runDir.WriteNormalizedTranscript(normalizedTranscript); err != nil {
-		_ = runDir.WriteErrorLog(fmt.Sprintf("normalized_artifact: %v", err))
-	}
-	if err := runDir.WriteNormalizationSummary(normSummary); err != nil {
-		_ = runDir.WriteErrorLog(fmt.Sprintf("normalization_summary: %v", err))
-	}
+	debugWrite("normalized transcript", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizedTranscript(normalizedTranscript) })
+	debugWrite("normalization summary", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizationSummary(normSummary) })
 
 	// Compute chunks after normalization
 	chunker := chunking.NewChunker(chunking.ChunkingConfig{
@@ -141,14 +157,12 @@ var processRunner = func(ctx context.Context, inv processInvocation, stdout io.W
 	chunkSummary := chunking.ComputeSummary(sections, chunkConfig)
 	chunkDetailedSummary := chunking.ComputeDetailedSummary(sections, chunkConfig)
 
-	if err := runDir.WriteChunkingSummary(&chunkDetailedSummary); err != nil {
-		_ = runDir.WriteErrorLog(fmt.Sprintf("chunking_summary: %v", err))
-	}
+	debugWrite("chunking summary", func(d *diagnostics.RunDirectory) error { return d.WriteChunkingSummary(&chunkDetailedSummary) })
 
 	workingTranscript := normalizedTranscript
 	var runOutput *runner.RunOutput
 	observations := llm.NewObservationCollector()
-	moduleFactory := modules.NewFactory(modules.Dependencies{Config: &inv.Config, Glossary: glossary, DiagnosticsDir: runDir.Path()})
+	moduleFactory := modules.NewFactory(modules.Dependencies{Config: &inv.Config, Glossary: glossary, DiagnosticsDir: debugPath})
 	if inv.LLMRuntime == nil {
 		return fail("runner_setup", fmt.Errorf("LLM runtime is required"), nil)
 	}
@@ -173,9 +187,10 @@ var processRunner = func(ctx context.Context, inv processInvocation, stdout io.W
 		ModuleSpecs: moduleSpecs,
 
 		ProposalLLMClient:        proposalLLMClient,
-		ProposalDiagnosticsDir:   runDir.Path(),
+		ProposalDiagnosticsDir:   debugPath,
 		ValidationLLMClient:      validationLLMClient,
-		ValidationDiagnosticsDir: runDir.Path(),
+		ValidationDiagnosticsDir: debugPath,
+		CaptureCandidates:        runDir != nil,
 	})
 	runnerResult.Observations = observations.Snapshot()
 	runnerResult.ObservationsAvailable = true
@@ -311,26 +326,24 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
 		ConfigVersion:  &version,
 	}
 
-	normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(ctx, inv, stdout)
+	normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(ctx, inv, stdout, stderr)
 	completedAt := time.Now().UTC()
 
 	if runErr != nil {
 		if runDir != nil && runOutput != nil {
-			writeExecutionArtifacts(runDir, runOutput, inv.Config)
+			writeExecutionArtifacts(runDir, runOutput, inv.Config, stderr)
 
-			_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
+			if debugErr := runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
 				RunDirectoryPath: runDir.Path(),
 				RunOutput:        runOutput,
-			}))
+			})); debugErr != nil {
+				fmt.Fprintf(stderr, "audita process: debug correction ledger: %s\n", llm.RedactError(debugErr, inv.Config))
+			}
 		}
 		errorPhase, errorMessage := extractErrorPhase(runErr)
 		errorMessage = llm.RedactError(fmt.Errorf("%s", errorMessage), inv.Config)
 		report := processreport.Build(processReportInput("failed", inv, runDir, startedAt, completedAt, errorMessage, errorPhase, nil, nil, runOutput))
 
-		if runDir != nil {
-			_ = runDir.WriteReport(report)
-		}
-
 		fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(runErr, inv.Config))
 		if runDir != nil {
 			fmt.Fprintf(stderr, "audita process: diagnostics: %s\n", runDir.Path())
@@ -339,29 +352,31 @@ func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) in
 	}
 
 	if runDir != nil && runOutput != nil {
-		writeExecutionArtifacts(runDir, runOutput, inv.Config)
+		writeExecutionArtifacts(runDir, runOutput, inv.Config, stderr)
 
-		_ = runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
+		if debugErr := runDir.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
 			RunDirectoryPath: runDir.Path(),
 			RunOutput:        runOutput,
-		}))
+		})); debugErr != nil {
+			fmt.Fprintf(stderr, "audita process: debug correction ledger: %s\n", llm.RedactError(debugErr, inv.Config))
+		}
 	}
 
 	report := processreport.Build(processReportInput("success", inv, runDir, startedAt, completedAt, "", "", normSummary, chunkSummary, runOutput))
 
-	if runDir != nil {
-		_ = runDir.WriteReport(report)
-	}
-
 	return 0
 }
 
-func writeExecutionArtifacts(runDir *diagnostics.RunDirectory, output *runner.RunOutput, cfg config.Config) {
+func writeExecutionArtifacts(runDir *diagnostics.RunDirectory, output *runner.RunOutput, cfg config.Config, stderr io.Writer) {
 	if !output.ObservationsAvailable {
 		return
 	}
-	_ = runDir.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations))
-	_ = runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, processreport.BuildUtilization(output, cfg.Backends))
+	if err := runDir.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations)); err != nil {
+		fmt.Fprintf(stderr, "audita process: debug model execution: %s\n", llm.RedactError(err, cfg))
+	}
+	if err := runDir.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, processreport.BuildUtilization(output, cfg.Backends)); err != nil {
+		fmt.Fprintf(stderr, "audita process: debug utilization: %s\n", llm.RedactError(err, cfg))
+	}
 }
 
 func runConfig(ctx context.Context, args []string, stdout, stderr io.Writer) int {