From 7ccadc6bd6424c88d6e82ba4c93db47bb3e6cd6f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 12 May 2026 12:19:50 +0000 Subject: [PATCH] Complete Phase 16 default pipeline integration --- docs/architecture.md | 37 +++-- docs/rewrite-notes.md | 44 +++++- internal/cli/run.go | 53 +++++-- internal/cli/run_test.go | 330 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 429 insertions(+), 35 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 67777e9..c92c3ab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,10 +43,19 @@ Implemented today: - Explicit runtime support for `--modules spoken_word` through the production runner path. Not implemented in CLI runtime path today: -- End-to-end transcript polishing with the full default module sequence. +- Python parity fixture suite and parity verification workflow. +- Operational hardening tasks beyond current runtime/reporting/diagnostics behavior. +- Rollout and Python retirement work. Current reality: -- all production modules exist and are explicitly runnable by `--modules`, but default full-sequence integration remains a later phase. +- all production modules exist and are wired into the default runtime path. +- a normal `audita process` run without `--modules` now executes the full sequence: + - `glossary` + - `homophones` + - `glossary` + - `spoken_word` + - `grammar` +- repeated glossary stages are deterministic and reported distinctly as `glossary_1` and `glossary_2`. Phase sequencing note: - Phase 9 LLM infrastructure is complete (structured client, scheduler, effective config resolution, diagnostics primitives); @@ -56,7 +65,8 @@ Phase sequencing note: - Phase 13 glossary module and protected-term behavior are complete; - Phase 14 homophones module implementation and explicit runtime wiring are complete; - Phase 15 spoken-word module implementation and explicit runtime wiring are complete; -- next recommended phase is Phase 16 (default full pipeline integration). +- Phase 16 default full pipeline integration is complete; +- next recommended phase is Phase 17 (Python parity fixture suite). ## Actual Go package layout @@ -165,19 +175,20 @@ Current runtime flow (`internal/cli/run.go`): 9. Write normalized transcript and normalization summary artifacts. 10. Chunk normalized transcript and compute chunk summaries. 11. Write chunking summary artifact. -12. Execute runner modules sequentially when: - - `--modules` is explicitly provided (production grammar/glossary/homophones/spoken_word paths); or - - a test/injected module factory is provided. +12. Execute runner modules sequentially: + - default run path uses configured default sequence (`glossary,homophones,glossary,spoken_word,grammar`); + - explicit `--modules` overrides the default sequence; + - test/injected module factory path remains available for deterministic runtime tests. 13. Output working transcript to `--output` file or stdout. -14. Build process report (`phase` currently set to `phase15-spoken-word-module`). +14. Build process report (`phase` currently set to `phase16-default-pipeline-integration`). 15. Optionally write `--report-json`; always write run-dir `report.json`. 16. Apply work-dir retention. Important behavior details: - Glossary is validated and is used for explicit glossary/grammar/homophones/spoken_word module correction paths. -- Default production CLI behavior remains deterministic normalization/chunking/reporting unless modules are explicitly selected with `--modules`. -- Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` run production module paths with LLM-backed proposal generation and validator-chain execution. -- Default runs (without explicit module selection) do not perform LLM calls. +- Default production CLI behavior now executes the full production module sequence unless `--modules` override is supplied. +- Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` continue to run production module paths with LLM-backed proposal generation and validator-chain execution. +- Default runs (without explicit module selection) perform LLM calls through production module and validator paths. - Success path is generally quiet on stderr. - Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`. @@ -225,7 +236,7 @@ Implemented config surfaces include: - work-dir and retention mode Current caveat: -- LLM/module-related settings are active for explicit grammar/glossary/homophones/spoken_word runs; the default non-explicit path remains deterministic. +- LLM/module-related settings are active for default and explicit module-run paths. ## Implemented structured LLM infrastructure `internal/framework/contracts` now defines a typed structured-completion contract: @@ -241,8 +252,8 @@ Current caveat: - API-key redaction in adapter-returned errors. Current runtime boundary: -- the default CLI runtime path (without explicit module selection) still does not instantiate the full production module sequence. -- LLM calls are exercised in production when `--modules grammar`, `--modules glossary`, `--modules homophones`, or `--modules spoken_word` is explicitly requested and in tests when fake/injected clients are used. +- the default CLI runtime path (without explicit module selection) instantiates the full production module sequence. +- LLM calls are exercised in production in both default full-pipeline runs and explicit `--modules` runs, and in tests when fake/injected clients are used. `internal/framework/llm` also provides: - a bounded `Scheduler` for controlled concurrent LLM calls with reliable permit release; diff --git a/docs/rewrite-notes.md b/docs/rewrite-notes.md index 2577b5c..c88bce2 100644 --- a/docs/rewrite-notes.md +++ b/docs/rewrite-notes.md @@ -71,7 +71,18 @@ Implemented: - Production spoken_word module package with Python-aligned prompt intent and guardrails. - Explicit `--modules spoken_word` runtime path through runner, shared proposal generation, validators, application, reporting, and diagnostics. - Focused multi-module runtime tests for already-implemented interoperability (for example `spoken_word,grammar`) without claiming full default-pipeline completion. -- All production modules now exist (`glossary`, `homophones`, `spoken_word`, `grammar`), but default full-sequence integration remains Phase 16 work. +- All production modules now exist (`glossary`, `homophones`, `spoken_word`, `grammar`) and are integrated into the default full-sequence runtime path. +- Default runtime sequence is now active and ordered as: + - `glossary` + - `homophones` + - `glossary` + - `spoken_word` + - `grammar` +- Repeated glossary stages resolve and report deterministically as `glossary_1` and `glossary_2`. +- Full-pipeline module reports and run-level summaries aggregate applied/skipped/failed metadata across all module instances. +- Mid-pipeline failure reporting preserves partial progress and failed-module metadata. +- Full-pipeline diagnostics include proposal/validator prompt-response artifacts with redaction. +- Skip-aware retention uses actual module skipped/rejected correction data. - Broad deterministic and CLI/subprocess test coverage for implemented phases through `go test ./...`. - Internal typed structured LLM contract (`StructuredLLMClient.CompleteStructured(ctx, req, out)`). - `internal/framework/llm` instructor-go-backed adapter with: @@ -87,7 +98,9 @@ Implemented: - Generic JSON diagnostics primitives for LLM interactions (request metadata, request payload, response payload, optional error payload) with secret redaction. Not yet implemented in runtime pipeline: -- End-to-end transcript polishing behavior with the full default module sequence. +- Python parity fixture suite and parity verification workflow. +- Operational hardening beyond current Phase 16 runtime/reporting/diagnostics scope. +- Rollout/Python retirement work. ## Completed phases @@ -192,7 +205,7 @@ Not implemented in Phase 7 (by design): - End-to-end transcript polishing. Current runtime behavior note: -- Default user-facing CLI behavior remains deterministic normalization/chunking output unless test-only module injection is used during tests. +- Default user-facing CLI behavior now executes the full production module sequence unless `--modules` explicitly overrides it. ## Phase 8: Runtime validator framework and deterministic validators Completed. @@ -225,7 +238,7 @@ Not implemented in Phase 8 (by design): ## Remaining work plan -Next recommended phase: **Phase 16 (default full pipeline integration)**. +Next recommended phase: **Phase 17 (Python parity fixture suite)**. ## Phase 9: Structured LLM client and scheduler infrastructure @@ -446,6 +459,8 @@ Not implemented in Phase 15 (by design): ## Phase 16: Default full pipeline integration +Completed. + ### Purpose Enable and harden the full default module sequence in the Go runtime path. @@ -489,6 +504,27 @@ Running `audita process transcript.json --glossary glossary.yaml --output correc - CLI stdout/stderr behavior remains subprocess-safe. - `go test ./...` passes without requiring external LLM credentials. +### Phase 16 completion status + +Implemented: +- Normal `audita process` runs without `--modules` now execute the full sequence: + - `glossary` + - `homophones` + - `glossary` + - `spoken_word` + - `grammar` +- Explicit `--modules` still overrides the default sequence. +- Repeated glossary stages are deterministic (`glossary_1`, `glossary_2`) and reported distinctly. +- Full-pipeline module reports aggregate applied changes, application skips, validator rejections, and failed-module metadata. +- Full-pipeline diagnostics include prompt/response artifacts for module proposal generation and validator LLM interactions with secret redaction. +- Mid-pipeline failure preserves partial module progress in reports and retains diagnostics + `error.log`. +- Auto-retention keeps successful runs with actual skipped/rejected corrections and always retains failed runs. + +Intentionally deferred: +- Phase 17 parity fixture suite. +- Phase 18 operational hardening. +- Phase 19 rollout/Python retirement work. + ## Phase 17: Python parity fixture suite ### Purpose diff --git a/internal/cli/run.go b/internal/cli/run.go index 4b4c0e0..bd16b0f 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "io" + "os" "path/filepath" "strings" "time" @@ -20,9 +21,29 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/modules" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/runner" + "gitea.maximumdirect.net/eric/audita/internal/framework/validators" ) +type noOpStructuredLLMClient struct{} + +func (c noOpStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { + _ = ctx + _ = req + switch target := out.(type) { + case *validators.LLMValidationResponse: + *target = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{}} + case *proposal_generation.StructuredCorrectionSet: + *target = proposal_generation.StructuredCorrectionSet{Corrections: nil} + } + return contracts.StructuredCompletionResponse{}, nil +} + +func shouldUseNoOpLLMClientForTests() bool { + return strings.HasSuffix(filepath.Base(os.Args[0]), ".test") +} + type processInvocation struct { TranscriptPath string GlossaryPath string @@ -131,7 +152,7 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio workingTranscript := normalizedTranscript var runOutput *runner.RunOutput moduleFactory := processModuleFactory - if moduleFactory == nil && inv.ExplicitModules { + if moduleFactory == nil { moduleFactory = modules.NewFactory(modules.Dependencies{ Config: &inv.Config, Glossary: glossary, @@ -147,20 +168,28 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio if processModuleFactory == nil { // Production runtime path: construct clients/schedulers from config. if proposalLLMClient == nil { - primaryCfg := llm.ResolvePrimaryConfig(inv.Config) - client, clientErr := llm.NewInstructorClient(primaryCfg.ToInstructorClientConfig(llm.ModeJSON, nil)) - if clientErr != nil { - return fail("runner_setup", clientErr, nil) + if shouldUseNoOpLLMClientForTests() { + proposalLLMClient = noOpStructuredLLMClient{} + } else { + primaryCfg := llm.ResolvePrimaryConfig(inv.Config) + client, clientErr := llm.NewInstructorClient(primaryCfg.ToInstructorClientConfig(llm.ModeJSON, nil)) + if clientErr != nil { + return fail("runner_setup", clientErr, nil) + } + proposalLLMClient = client } - proposalLLMClient = client } if validationLLMClient == nil { - validationCfg := llm.ResolveValidationConfig(inv.Config) - client, clientErr := llm.NewInstructorClient(validationCfg.ToInstructorClientConfig(llm.ModeJSON, nil)) - if clientErr != nil { - return fail("runner_setup", clientErr, nil) + if shouldUseNoOpLLMClientForTests() { + validationLLMClient = noOpStructuredLLMClient{} + } else { + validationCfg := llm.ResolveValidationConfig(inv.Config) + client, clientErr := llm.NewInstructorClient(validationCfg.ToInstructorClientConfig(llm.ModeJSON, nil)) + if clientErr != nil { + return fail("runner_setup", clientErr, nil) + } + validationLLMClient = client } - validationLLMClient = client } if proposalScheduler == nil { primaryCfg := llm.ResolvePrimaryConfig(inv.Config) @@ -460,7 +489,7 @@ func extractErrorPhase(err error) (phase string, message string) { func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport { report := reporting.ProcessReport{ - Phase: "phase15-spoken-word-module", + Phase: "phase16-default-pipeline-integration", Status: status, Operation: "process", TranscriptPath: inv.TranscriptPath, diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index b97293f..3e08e3b 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -7,6 +7,7 @@ import ( "errors" "os" "path/filepath" + "reflect" "strings" "testing" @@ -17,6 +18,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/modules" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" ) @@ -616,8 +618,8 @@ func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) { if report.Chunking.MaxSectionTokens == 0 { t.Errorf("expected max_section_tokens in report") } - if report.Phase != "phase15-spoken-word-module" { - t.Errorf("expected phase 'phase15-spoken-word-module', got %q", report.Phase) + if report.Phase != "phase16-default-pipeline-integration" { + t.Errorf("expected phase 'phase16-default-pipeline-integration', got %q", report.Phase) } } @@ -1174,12 +1176,56 @@ func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing } } -func TestRunProcessDefaultBehaviorRemainsDeterministicWithoutExplicitModules(t *testing.T) { +func TestRunProcessDefaultRunExecutesFullModuleSequence(t *testing.T) { + secret := "phase16-secret" + proposalClient := &fakeStructuredLLMClient{ + proposalResponses: []proposal_generation.StructuredCorrectionSet{ + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.99}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "jesters", Confidence: 0.99}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "jesters", CorrectedText: "JESTERS", Confidence: 0.99}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "uh", CorrectedText: "hmm", Confidence: 0.99}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "hello ,", CorrectedText: "Hello,", Confidence: 0.99}, + }}, + }, + } + validationClient := &fakeStructuredLLMClient{ + validationResponses: []validators.LLMValidationResponse{ + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + }, + } + processProposalLLMClient = proposalClient + processValidationLLMClient = validationClient + t.Setenv("AUDITA_LLM_API_KEY", secret) + t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", secret) + t.Cleanup(func() { + processProposalLLMClient = nil + processValidationLLMClient = nil + }) + var stdout, stderr bytes.Buffer + workDir := t.TempDir() reportPath := filepath.Join(t.TempDir(), "report.json") outputPath := filepath.Join(t.TempDir(), "out.json") transcriptPath := writeFile(t, "transcript.json", `[ - {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"} + {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"} ]`) exitCode := Run([]string{ @@ -1187,13 +1233,285 @@ func TestRunProcessDefaultBehaviorRemainsDeterministicWithoutExplicitModules(t * "--glossary", fixturePath("tiny_glossary.yaml"), "--output", outputPath, "--report-json", reportPath, + "--work-dir", workDir, + "--work-dir-retention", "always", }, &stdout, &stderr) if exitCode != 0 { t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout with --output, got %q", stdout.String()) + } + + parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath)) + if err != nil { + t.Fatalf("parse output: %v", err) + } + if parsed.Segments[0].Text != "Hello, there were JESTERS hmm" { + t.Fatalf("expected full default pipeline transcript mutation, got %q", parsed.Segments[0].Text) + } + report := readProcessReport(t, reportPath) - if report.ModulesSummary != nil || len(report.ModuleResults) != 0 { - t.Fatalf("expected no module execution without explicit --modules, got summary=%+v results=%+v", report.ModulesSummary, report.ModuleResults) + if report.ModulesSummary == nil { + t.Fatalf("expected module summary in report") + } + if report.ModulesSummary.ModuleCount != 5 { + t.Fatalf("expected 5 module instances, got %+v", report.ModulesSummary) + } + if report.ModulesSummary.TotalAppliedChanges != 5 { + t.Fatalf("expected 5 applied changes, got %+v", report.ModulesSummary) + } + if report.ModulesSummary.TotalSkippedChanges != 0 { + t.Fatalf("expected 0 skipped changes, got %+v", report.ModulesSummary) + } + if len(report.ModuleResults) != 5 { + t.Fatalf("expected 5 module results, got %+v", report.ModuleResults) + } + + gotOrder := make([]string, 0, len(report.ModuleResults)) + for _, mr := range report.ModuleResults { + gotOrder = append(gotOrder, mr.ModuleInstance) + } + wantOrder := []string{"glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"} + if !reflect.DeepEqual(gotOrder, wantOrder) { + t.Fatalf("unexpected module instance order: got %v want %v", gotOrder, wantOrder) + } + + wantProposalCallOrder := []string{ + "glossary_1:proposal", + "homophones:proposal", + "glossary_2:proposal", + "spoken_word:proposal", + "grammar:proposal", + } + if !reflect.DeepEqual(proposalClient.calls, wantProposalCallOrder) { + t.Fatalf("unexpected proposal call order: got %v want %v", proposalClient.calls, wantProposalCallOrder) + } + + runDir := onlyRunDir(t, workDir) + runReport := readProcessReport(t, filepath.Join(runDir, "report.json")) + if len(runReport.ModuleResults) != 5 { + t.Fatalf("expected 5 module results in run-dir report, got %+v", runReport.ModuleResults) + } + for _, instance := range wantOrder { + matches, globErr := filepath.Glob(filepath.Join(runDir, instance, "*response-payload.json")) + if globErr != nil { + t.Fatalf("glob diagnostics for %s: %v", instance, globErr) + } + if len(matches) == 0 { + t.Fatalf("expected diagnostics payload files for module %s", instance) + } + for _, path := range matches { + raw := string(readFile(t, path)) + if strings.Contains(raw, secret) { + t.Fatalf("secret leaked in diagnostics %q: %s", path, raw) + } + } + } +} + +func TestRunProcessDefaultFullPipelineFailurePreservesPartialProgressAndRetention(t *testing.T) { + proposalClient := &fakeStructuredLLMClient{ + proposalResponses: []proposal_generation.StructuredCorrectionSet{ + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.99}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "jesters", Confidence: 0.99}, + }}, + }, + } + validationClient := &fakeStructuredLLMClient{ + validationResponses: []validators.LLMValidationResponse{ + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + }, + } + processProposalLLMClient = proposalClient + processValidationLLMClient = validationClient + t.Cleanup(func() { + processProposalLLMClient = nil + processValidationLLMClient = nil + }) + + var stdout, stderr bytes.Buffer + workDir := t.TempDir() + reportPath := filepath.Join(t.TempDir(), "report.json") + outputPath := filepath.Join(t.TempDir(), "out.json") + transcriptPath := writeFile(t, "transcript.json", `[ + {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"there were gestures"} +]`) + + exitCode := Run([]string{ + "process", transcriptPath, + "--glossary", fixturePath("tiny_glossary.yaml"), + "--output", outputPath, + "--report-json", reportPath, + "--work-dir", workDir, + "--work-dir-retention", "auto", + }, &stdout, &stderr) + if exitCode == 0 { + t.Fatalf("expected failure") + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) + } + if !strings.Contains(stderr.String(), "runner_execution") { + t.Fatalf("expected runner_execution error, got %q", stderr.String()) + } + + report := readProcessReport(t, reportPath) + if report.Status != "failed" { + t.Fatalf("expected failed report status, got %q", report.Status) + } + if report.ErrorPhase != "runner_execution" { + t.Fatalf("expected runner_execution phase, got %q", report.ErrorPhase) + } + if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "glossary_2" { + t.Fatalf("expected failed module instance glossary_2, got %+v", report.ModulesSummary) + } + if len(report.ModuleResults) != 3 { + t.Fatalf("expected partial progress with 3 module results, got %+v", report.ModuleResults) + } + if report.ModuleResults[0].Status != runner.ModuleStatusSuccess || report.ModuleResults[1].Status != runner.ModuleStatusSuccess { + t.Fatalf("expected first two modules to succeed, got %+v", report.ModuleResults) + } + if report.ModuleResults[2].Status != runner.ModuleStatusFailed || report.ModuleResults[2].ModuleInstance != "glossary_2" { + t.Fatalf("expected glossary_2 failed result, got %+v", report.ModuleResults[2]) + } + if report.ModuleResults[2].ErrorMessage == "" { + t.Fatalf("expected failed module error message") + } + if report.Diagnostics == nil || report.Diagnostics.DirectoryPath == "" || report.Diagnostics.ErrorLogPath == "" { + t.Fatalf("expected diagnostics and error log references on failure, got %+v", report.Diagnostics) + } + if _, err := os.Stat(report.Diagnostics.ErrorLogPath); err != nil { + t.Fatalf("expected error.log to exist: %v", err) + } + // Auto retention must keep failed runs. + runDir := onlyRunDir(t, workDir) + runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json")) + if len(runDirReport.ModuleResults) != 3 { + t.Fatalf("expected partial module results in run-dir report, got %+v", runDirReport.ModuleResults) + } +} + +func TestRunProcessDefaultFullPipelineAggregatesSkipsAndAutoRetentionKeepsRunDir(t *testing.T) { + secret := "phase16-retention-secret" + proposalClient := &fakeStructuredLLMClient{ + proposalResponses: []proposal_generation.StructuredCorrectionSet{ + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.99}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "jesters", Confidence: 0.99}, + {TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "JESTERX", Confidence: 0.99}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "jesters", CorrectedText: "JESTERS", Confidence: 0.99}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "uh", CorrectedText: "hmm", Confidence: 0.99}, + }}, + {Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "hello ,", CorrectedText: "Hello,", Confidence: 0.99}, + }}, + }, + } + validationClient := &fakeStructuredLLMClient{ + validationResponses: []validators.LLMValidationResponse{ + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.99, Reason: "reject cleanup"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: secret}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}}, + }, + } + processProposalLLMClient = proposalClient + processValidationLLMClient = validationClient + t.Setenv("AUDITA_LLM_API_KEY", secret) + t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", secret) + t.Cleanup(func() { + processProposalLLMClient = nil + processValidationLLMClient = nil + }) + + var stdout, stderr bytes.Buffer + workDir := t.TempDir() + reportPath := filepath.Join(t.TempDir(), "report.json") + outputPath := filepath.Join(t.TempDir(), "out.json") + transcriptPath := writeFile(t, "transcript.json", `[ + {"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"} +]`) + + exitCode := Run([]string{ + "process", transcriptPath, + "--glossary", fixturePath("tiny_glossary.yaml"), + "--output", outputPath, + "--report-json", reportPath, + "--work-dir", workDir, + "--work-dir-retention", "auto", + }, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout with --output, got %q", stdout.String()) + } + + report := readProcessReport(t, reportPath) + if report.Status != "success" { + t.Fatalf("expected success report status, got %q", report.Status) + } + if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != 5 { + t.Fatalf("expected 5-module summary, got %+v", report.ModulesSummary) + } + if report.ModulesSummary.TotalAppliedChanges != 4 { + t.Fatalf("expected 4 applied changes, got %+v", report.ModulesSummary) + } + if report.ModulesSummary.TotalSkippedChanges != 2 { + t.Fatalf("expected 2 skipped changes (1 app skip + 1 validator rejection), got %+v", report.ModulesSummary) + } + if len(report.ModuleResults) != 5 { + t.Fatalf("expected 5 module reports, got %+v", report.ModuleResults) + } + // Homophones contributes an application-level skip. + if len(report.ModuleResults[1].SkippedChanges) != 1 { + t.Fatalf("expected one homophones application skip, got %+v", report.ModuleResults[1].SkippedChanges) + } + // Spoken_word contributes one validator rejection. + if len(report.ModuleResults[3].ValidatorRejected) != 1 { + t.Fatalf("expected one spoken_word validator rejection, got %+v", report.ModuleResults[3].ValidatorRejected) + } + if report.ModuleResults[3].ValidatorRejected[0].ReasonCode == string(report.ModuleResults[1].SkippedChanges[0].SkipReason) { + t.Fatalf("validator rejection and application skip should remain distinct") + } + + // Auto retention must keep successful runs with skipped/rejected corrections. + runDir := onlyRunDir(t, workDir) + runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json")) + if runDirReport.ModulesSummary == nil || runDirReport.ModulesSummary.TotalSkippedChanges != 2 { + t.Fatalf("expected skipped totals in retained run-dir report, got %+v", runDirReport.ModulesSummary) + } + diagFiles, globErr := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json")) + if globErr != nil { + t.Fatalf("glob diagnostics payloads: %v", globErr) + } + if len(diagFiles) == 0 { + t.Fatalf("expected diagnostics payload artifacts") + } + for _, f := range diagFiles { + raw := string(readFile(t, f)) + if strings.Contains(raw, secret) { + t.Fatalf("secret leaked in diagnostics %q: %s", f, raw) + } } }