From a3655f55407dadd8e4d8f8959ebfe0025b2bbb5f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 23 May 2026 10:07:06 -0500 Subject: [PATCH] Make module-stage LLM handling resilient and report warnings --- README.md | 10 +- cmd/audita/main_integration_test.go | 21 ++- docs/architecture/architecture.md | 32 +++-- docs/architecture/diagnostics.md | 6 + docs/architecture/public-contract.md | 3 + docs/architecture/validators.md | 16 ++- docs/release-checklist.md | 4 + docs/roadmap/publish.md | 72 ++++++++++ internal/cli/run.go | 2 + internal/cli/run_test.go | 126 +++++++++++------- internal/cli/subprocess_test_hooks.go | 23 ++-- internal/core/reporting/report.go | 26 ++-- .../chunk_proposal_composition_test.go | 8 +- internal/framework/contracts/contracts.go | 8 +- .../framework/contracts/contracts_test.go | 14 +- internal/framework/modules/registry_test.go | 4 +- .../framework/proposal_generation/generate.go | 54 +++++++- .../proposal_generation/generate_test.go | 39 +++++- internal/framework/proposals/apply.go | 2 + internal/framework/proposals/preview.go | 7 + internal/framework/proposals/preview_test.go | 36 +++++ internal/framework/proposals/proposal.go | 3 - internal/framework/proposals/proposal_test.go | 10 +- internal/framework/runner/runner.go | 79 +++++++++-- internal/framework/runner/runner_test.go | 37 ++--- .../framework/validators/deterministic.go | 44 +++++- .../framework/validators/llm_validators.go | 123 ++++++++++++++++- .../validators/llm_validators_test.go | 95 ++++++++++--- internal/framework/validators/models.go | 25 ++-- .../framework/validators/validators_test.go | 47 ++++++- internal/framework/warnings/warnings.go | 18 +++ internal/modules/glossary/module.go | 11 +- internal/modules/glossary/module_test.go | 3 +- internal/modules/grammar/module.go | 11 +- internal/modules/grammar/module_test.go | 3 +- internal/modules/homophones/module.go | 11 +- internal/modules/homophones/module_test.go | 3 +- internal/modules/spoken_word/module.go | 11 +- internal/modules/spoken_word/module_test.go | 3 +- internal/validators/chains.go | 4 + .../validators/proposal_shape/validator.go | 13 ++ internal/validators/registry.go | 3 + internal/validators/registry_test.go | 3 + 43 files changed, 856 insertions(+), 217 deletions(-) create mode 100644 docs/roadmap/publish.md create mode 100644 internal/framework/warnings/warnings.go create mode 100644 internal/validators/proposal_shape/validator.go diff --git a/README.md b/README.md index 912f5d7..7f50361 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Pipeline behavior includes: - conservative spoken-word dysfluency cleanup with semantic guardrails - grammar/punctuation/capitalization/formatting cleanup - validator-chain enforcement before application +- malformed module-stage LLM payloads degrade to warnings/rejections instead of failing the run - run reports and diagnostics artifacts with secret redaction ## Build and Install @@ -130,6 +131,7 @@ audita process transcript.json \ - Without `--output`, stdout contains transcript JSON only on success. - `--report-json` writes a file and is never printed to stdout. - stderr is human-readable diagnostics/errors. +- successful runs remain quiet on stderr even when module warnings are recorded in report/diagnostics artifacts. For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md). @@ -149,10 +151,10 @@ audita config print-effective --config audita.yml ``` For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md). -For output-schema details, see [`docs/output-schemas.md`](docs/output-schemas.md). -For built-in validator keys and chain definitions, see [`docs/validators.md`](docs/validators.md). -For embedded prompt assets and prompt metadata behavior, see [`docs/prompts.md`](docs/prompts.md). -For CLI/process compatibility guarantees, see [`docs/public-contract.md`](docs/public-contract.md). +For output-schema details, see [`docs/architecture/output-schemas.md`](docs/architecture/output-schemas.md). +For built-in validator keys and chain definitions, see [`docs/architecture/validators.md`](docs/architecture/validators.md). +For embedded prompt assets and prompt metadata behavior, see [`docs/architecture/prompts.md`](docs/architecture/prompts.md). +For CLI/process compatibility guarantees, see [`docs/architecture/public-contract.md`](docs/architecture/public-contract.md). ### Modules diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index 0ac5d11..355e2a3 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -382,25 +382,22 @@ func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testin "--work-dir-retention", "always", ) - if result.exitCode == 0 { - t.Fatalf("expected nonzero exit code") + if result.exitCode != 0 { + t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr) } - if result.stdout != "" { - t.Fatalf("expected empty stdout on failure, got %q", result.stdout) + if !json.Valid([]byte(result.stdout)) { + t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout) } - if !strings.Contains(result.stderr, "runner_execution") { - t.Fatalf("expected runner_execution failure, got %q", result.stderr) - } - if !strings.Contains(result.stderr, "diagnostics:") { - t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr) + if result.stderr != "" { + t.Fatalf("expected empty stderr on success, got %q", result.stderr) } report := readFile(t, reportPath) if !json.Valid(report) { - t.Fatalf("expected valid failure report JSON") + t.Fatalf("expected valid success report JSON") } runDir := onlyRunDir(t, workDir) - if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { - t.Fatalf("expected error.log, got: %v", err) + if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) { + t.Fatalf("did not expect error.log, got: %v", err) } } diff --git a/docs/architecture/architecture.md b/docs/architecture/architecture.md index 3e2797e..96b76e4 100644 --- a/docs/architecture/architecture.md +++ b/docs/architecture/architecture.md @@ -133,11 +133,16 @@ internal/framework/validators/ llm_batching.go llm_validators.go +internal/framework/warnings/ + warnings.go + internal/validators/ metadata/ metadata.go registry.go chains.go + proposal_shape/ + validator.go confidence_threshold/ validator.go original_text_presence/ @@ -242,6 +247,7 @@ Important behavior details: - Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` continue to run production module paths with LLM-backed proposal generation and validator-chain execution. - Default runs (without explicit module selection) perform LLM calls through production module and validator paths. - Success path is generally quiet on stderr. +- Malformed module-stage LLM payloads degrade to validator rejections and module warnings instead of aborting the run. - Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`. ## Implemented data contracts @@ -539,20 +545,20 @@ Validator execution classification metadata: Stable built-in validator keys: - deterministic: + - `proposal_shape` - `confidence_threshold` - `original_text_presence` - - `non_empty_corrected_text` + - `non_empty_corrected_text` (historical key name; current semantics reject empty resulting segment text) - `no_effect` - `protected_terms` - LLM-backed: - `spoken_form_plausibility` - `meaning_reversal_review` - `editorial_review` - - `grammar_review` - - `spoken_word_review` Built-in module chains: - `glossary`: + - `proposal_shape` - `no_effect` - `original_text_presence` - `confidence_threshold` @@ -561,6 +567,7 @@ Built-in module chains: - `spoken_form_plausibility` - `meaning_reversal_review` - `homophones`: + - `proposal_shape` - `no_effect` - `original_text_presence` - `confidence_threshold` @@ -569,20 +576,22 @@ Built-in module chains: - `spoken_form_plausibility` - `meaning_reversal_review` - `spoken_word`: + - `proposal_shape` - `no_effect` - `original_text_presence` - `confidence_threshold` - `protected_terms` - `non_empty_corrected_text` - - `spoken_word_review` + - `editorial_review` - `meaning_reversal_review` - `grammar`: + - `proposal_shape` - `no_effect` - `original_text_presence` - `confidence_threshold` - `protected_terms` - `non_empty_corrected_text` - - `grammar_review` + - `editorial_review` - `meaning_reversal_review` 1.0 boundary: @@ -595,12 +604,14 @@ Built-in module chains: - prompt builders for: - spoken-form plausibility - meaning reversal detection - - editorial review - - grammar review - - spoken-word review +- editorial review +- grammar review +- spoken-word review - deterministic batching by `validation_max_prompt_tokens`; -- strict cardinality validation of structured LLM decisions (missing/duplicate/unknown indexes fail); -- safe failure behavior for malformed/invalid structured responses. +- strict cardinality validation of synthesized validator decision sets; +- malformed validator payloads reject only the affected batch with warnings; +- oversized single-proposal validator inputs reject only the affected proposal; +- transport/provider/runtime LLM call failures remain fatal. `internal/framework/runner` wires LLM validators into existing validator chains using: - the internal structured LLM client abstraction (`contracts.StructuredLLMClient`); @@ -726,6 +737,7 @@ Current process reports include diagnostics metadata references for: Current process reports also include: - module-level results (when runner modules execute), including applied/skipped proposal changes; - run-level module summary totals and failed module instance metadata. +- module-level warning records for malformed proposal-generation payloads and malformed validator batches. - module-level validator decisions and validator rejections. - optional decision-level diagnostic artifact paths for validator LLM interactions when available. - stable validator keys in `validator_name` fields for validator decisions/rejections. diff --git a/docs/architecture/diagnostics.md b/docs/architecture/diagnostics.md index 594c982..a96bf3a 100644 --- a/docs/architecture/diagnostics.md +++ b/docs/architecture/diagnostics.md @@ -63,6 +63,7 @@ Ledger records are flattened review entries derived from module results and incl - deterministic and LLM validator decision snapshots using stable validator keys. Validator rejection and proposal-application skip are distinct dispositions. +Module warnings are reported in module results and diagnostics metadata, but do not create standalone correction-ledger rows. ## Report references @@ -71,6 +72,8 @@ Validator rejection and proposal-application skip are distinct dispositions. - correction ledger artifact; - existing transcript/normalization/chunking/invocation/effective-config artifacts. +Module report entries also include warning records for malformed proposal-generation payloads and malformed validator batches. + ## Retention behavior Run-directory retention follows configured policy: @@ -93,6 +96,9 @@ When debugging: - validator rejections: - inspect `correction-ledger.json` rejected entries and matching validator decisions; - inspect validator response diagnostics payloads. +- module warnings: + - inspect module `warnings` entries in `report.json` or `--report-json`; + - follow any diagnostic artifact path on the warning to the recorded error/response payload. - application skips: - inspect `correction-ledger.json` skipped entries and skip reason codes; - compare with validator decisions to distinguish validation rejection vs apply-time skip. diff --git a/docs/architecture/public-contract.md b/docs/architecture/public-contract.md index 2d65f48..ac253ad 100644 --- a/docs/architecture/public-contract.md +++ b/docs/architecture/public-contract.md @@ -93,6 +93,7 @@ Current values: `--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata. Validator decision/rejection records in reports use stable validator keys in `validator_name`. +Module results may also include warning records for malformed module-stage LLM payloads. Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds. ## Diagnostics directory behavior @@ -123,6 +124,7 @@ Success behavior: - with `--output`, stdout is empty - without `--output`, stdout contains only transcript JSON in selected output schema - report JSON is not written to stdout +- success stderr remains empty even when reports/diagnostics contain module warnings Failure behavior: - stderr contains human-readable error summary @@ -149,6 +151,7 @@ Config files should reference secrets via environment variable names (`api_key_e ## Compatibility and deprecation policy - Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract. +- Existing stable validator keys remain public contract values even when validator semantics are refined. - Compatibility inputs (legacy flags/env aliases) may remain during transition windows. - Any planned removal or behavior change should include clear compatibility notes and migration guidance. diff --git a/docs/architecture/validators.md b/docs/architecture/validators.md index d3a25ca..e801416 100644 --- a/docs/architecture/validators.md +++ b/docs/architecture/validators.md @@ -8,6 +8,7 @@ For LLM-backed validator prompt asset details, see [`docs/prompts.md`](prompts.m Built-in validator construction is package-owned under `internal/validators/`: - `internal/validators/confidence_threshold` +- `internal/validators/proposal_shape` - `internal/validators/original_text_presence` - `internal/validators/non_empty_corrected_text` - `internal/validators/no_effect` @@ -45,12 +46,14 @@ Current 1.0 boundary: ### Deterministic validators +- `proposal_shape` + - rejects malformed proposal fields before other validators run. - `confidence_threshold` - checks proposal confidence against module-specific configured threshold. - `original_text_presence` - ensures target segment exists and `original_text` exists in current working segment text. - `non_empty_corrected_text` - - rejects blank/whitespace-only `corrected_text`. + - rejects proposals whose previewed resulting segment text would be empty or whitespace-only. - `no_effect` - rejects proposals where `original_text == corrected_text`. - `protected_terms` @@ -71,6 +74,7 @@ Current 1.0 boundary: Current built-in chains resolved from `internal/validators/chains.go`: - `glossary` + - `proposal_shape` - `no_effect` - `original_text_presence` - `confidence_threshold` @@ -80,6 +84,7 @@ Current built-in chains resolved from `internal/validators/chains.go`: - `meaning_reversal_review` - `homophones` + - `proposal_shape` - `no_effect` - `original_text_presence` - `confidence_threshold` @@ -89,6 +94,7 @@ Current built-in chains resolved from `internal/validators/chains.go`: - `meaning_reversal_review` - `spoken_word` + - `proposal_shape` - `no_effect` - `original_text_presence` - `confidence_threshold` @@ -98,6 +104,7 @@ Current built-in chains resolved from `internal/validators/chains.go`: - `meaning_reversal_review` - `grammar` + - `proposal_shape` - `no_effect` - `original_text_presence` - `confidence_threshold` @@ -119,7 +126,9 @@ Both variants preserve existing behavior and report the stable key `protected_te - modules execute serially; - section proposal work can run concurrently within a module; - deterministic validators run before LLM-backed validators; -- malformed/missing/duplicate/unknown LLM validator decisions fail safely; +- malformed module proposal payloads are downgraded to section-scoped module warnings with zero proposals for the affected section rather than module failure; +- malformed/missing/duplicate/unknown LLM validator decisions reject the affected validator batch with warnings instead of failing the module; +- oversized single-proposal validator inputs reject only the affected proposal under that validator; - approved proposals are applied once per module after section work settles. ## Validator rejections vs proposal-application skips @@ -128,12 +137,15 @@ Both variants preserve existing behavior and report the stable key `protected_te - proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code. - proposal-application skip: - proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time). +- module warning: + - malformed proposal-generation payloads and malformed validator batches are recorded in module warning records and diagnostics without writing success stderr. These are separate outcomes and are reported separately. ## Reporting and diagnostics identity - report validator decision/rejection entries use stable validator keys in `validator_name`. +- report module results include warning records for malformed module-stage LLM payloads. - validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata. - correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip. diff --git a/docs/release-checklist.md b/docs/release-checklist.md index e35c216..23d40f1 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -40,6 +40,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate. - Verify structured response schemas are attached via `response_format.type=json_schema`. - Verify diagnostics metadata includes structured schema `id/version/name/sha256`. - Verify provider output is still locally decoded/validated before use. +- Verify malformed module-stage structured payloads degrade to warnings/rejections instead of failing the run. ## Report and diagnostics schema checks @@ -67,6 +68,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate. - Verify prompt metadata appears in LLM request metadata diagnostics: - `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`. - Verify stable validator keys appear in report decisions/rejections. +- Verify module warning records appear in reports for malformed proposal-generation payloads and malformed validator batches. - Verify built-in validator chains resolve and execute for default and explicit module runs. ## Utilization diagnostics checks @@ -96,6 +98,8 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate. ## Failure and cancellation checks - Verify controlled failure paths retain diagnostics and produce best-effort failure reports. +- Verify malformed proposal-generation payloads keep exit code `0`, keep stderr empty on success, and record warnings in reports/diagnostics. +- Verify malformed validator payloads reject only the affected batch and do not fail the module. - Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized. ## Release fixture/idempotence checks diff --git a/docs/roadmap/publish.md b/docs/roadmap/publish.md new file mode 100644 index 0000000..2380f28 --- /dev/null +++ b/docs/roadmap/publish.md @@ -0,0 +1,72 @@ +# Hard-Cutover Roadmap for Module-Stage LLM Resilience + +## Summary + +This roadmap captures the module-stage resilience work for Audita: + +- fail fast on initialization, configuration, schema, and other pre-module setup errors; +- remain resilient during module execution when LLM payloads are malformed or individual proposed corrections are invalid; +- reject bad corrections through validator/reporting paths instead of aborting the module or process; +- keep success stderr quiet; surface warnings only through report and diagnostics artifacts; +- use a hard cutover only, with no compatibility aliases or transitional code. + +Locked decisions: + +- keep the stable public validator key `non_empty_corrected_text`; +- change that validator’s behavior to mean “the resulting segment text must not be empty/whitespace-only after applying the proposal preview”; +- malformed LLM-validator batch payloads reject the entire affected batch under that validator and continue; +- proposal/validator transport failures, timeouts, and provider/runtime call failures remain fatal; +- malformed structured payloads are downgraded; non-malformed runtime call failures are not. + +## Stage 1: Proposal Intake Hardening + +- Stop treating invalid individual structured corrections as fatal during proposal generation. +- Preserve returned correction ordering and proposal-index assignment even when individual corrections are malformed. +- Allow `corrected_text == ""` when the resulting segment remains non-empty after previewed application. +- Downgrade malformed proposal-generation structured payloads into section-scoped warnings with zero proposals for that section. +- Keep proposal-generation transport/provider/runtime call failures fatal. + +Deterministic validation changes: + +- Add `proposal_shape` as a built-in deterministic validator and run it first in every built-in module chain. +- Reject malformed proposal fields with stable reason codes: + - `invalid_target_segment_id` + - `empty_original_text` + - `invalid_confidence` +- Keep `non_empty_corrected_text` as the stable validator key, but change its semantics to reject only `empty_resulting_segment`. +- Keep validator rejection and apply-time skip as distinct outcomes. + +## Stage 2: LLM Validator Resilience + +- Keep validator transport/provider/runtime call failures fatal. +- Downgrade malformed validator structured payloads into batch-scoped validator rejections plus module warnings. +- Downgrade oversized single-proposal validator inputs into per-proposal validator rejections plus module warnings. +- Preserve decision-cardinality enforcement as an internal invariant after malformed-payload degradation has synthesized complete decision sets. + +Stable reason codes introduced or relied upon by this cutover: + +- `empty_resulting_segment` +- `invalid_target_segment_id` +- `empty_original_text` +- `invalid_confidence` +- `validator_response_malformed` +- `validator_input_too_large` +- `proposal_response_malformed` + +## Stage 3: Reporting, Diagnostics, and Docs + +- Add module warning records to runner results and process reports. +- Keep correction-ledger entries per-correction only; do not add standalone warning rows. +- Record malformed proposal-generation and validator-batch warnings through report and diagnostics artifacts only. +- Keep successful runs quiet on stderr even when warnings are present. +- Update README and architecture/public-contract/diagnostics/validators/release-checklist docs to reflect the new behavior. + +## Acceptance Criteria + +- Empty `corrected_text` may delete words, but proposals that would blank the whole segment are rejected or skipped safely. +- Invalid proposal shape is rejected by validators, not by proposal generation. +- Malformed proposal-generation payloads succeed with warnings and zero proposals for the affected section. +- Malformed validator payloads reject only the affected validator batch and do not fail the module. +- Oversized single validator inputs reject only the affected proposal. +- Transport/provider/runtime LLM failures still fail the module and process. +- Successful runs with warnings still exit `0`, emit transcript output normally, keep stderr empty, and expose warnings in report/diagnostics artifacts. diff --git a/internal/cli/run.go b/internal/cli/run.go index 504b6e4..d40eeab 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -26,6 +26,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" + stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) type noOpStructuredLLMClient struct{} @@ -803,6 +804,7 @@ func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummar ReplacementPolicy: string(r.ReplacementPolicy), Status: r.Status, ProposalCount: r.ProposalCount, + Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...), ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions), ValidatorRejected: mapValidatorRejected(r.ValidatorRejected), AppliedChanges: r.AppliedChanges, diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index d21303c..7b152e1 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -1793,11 +1793,12 @@ type fakeModule struct { func (m fakeModule) Key() string { return m.key } func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m fakeModule) Validators() []contracts.Validator { return m.validators } -func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { if m.proposeF == nil { - return nil, nil + return contracts.ProposalResult{}, nil } - return m.proposeF(req) + proposalsOut, err := m.proposeF(req) + return contracts.ProposalResult{Proposals: proposalsOut}, err } type fakeValidator struct { @@ -2384,7 +2385,7 @@ func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testi } } -func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing.T) { +func TestRunProcessExplicitGrammarMalformedLLMOutputSucceedsWithWarning(t *testing.T) { processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")} t.Cleanup(func() { processProposalLLMClient = nil }) @@ -2403,22 +2404,32 @@ func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing "--work-dir-retention", "always", "--report-json", reportPath, }, &stdout, &stderr) - if exitCode == 0 { - t.Fatal("expected failure") + if exitCode != 0 { + t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) } - if stdout.Len() != 0 { - t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr on success, got %q", stderr.String()) } - if !strings.Contains(stderr.String(), "runner_execution") { - t.Fatalf("expected runner_execution error, got %q", stderr.String()) + parsed, err := schema.ParseTranscriptJSON(stdout.Bytes()) + if err != nil { + t.Fatalf("expected transcript stdout on success: %v", err) + } + if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" { + t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments) } runDir := onlyRunDir(t, workDir) - if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { - t.Fatalf("expected error.log on failed grammar run: %v", err) + if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) { + t.Fatalf("did not expect error.log on successful grammar run: %v", err) } report := readProcessReport(t, reportPath) - if report.Status != "failed" || report.ErrorPhase != "runner_execution" { - t.Fatalf("expected failed runner_execution report, got %+v", report) + if report.Status != "success" || report.ErrorPhase != "" { + t.Fatalf("expected successful report, got %+v", report) + } + if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 { + t.Fatalf("expected one module warning, got %+v", report.ModuleResults) + } + if report.ModuleResults[0].Warnings[0].ReasonCode != "proposal_response_malformed" { + t.Fatalf("unexpected warning: %+v", report.ModuleResults[0].Warnings[0]) } } @@ -3041,7 +3052,7 @@ func TestRunProcessExplicitGlossaryRepeatedStagesUseDeterministicInstanceNamesAn } } -func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testing.T) { +func TestRunProcessExplicitGlossaryMalformedLLMOutputSucceedsWithWarning(t *testing.T) { processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")} t.Cleanup(func() { processProposalLLMClient = nil }) @@ -3060,22 +3071,29 @@ func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testin "--work-dir-retention", "always", "--report-json", reportPath, }, &stdout, &stderr) - if exitCode == 0 { - t.Fatal("expected failure") + if exitCode != 0 { + t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) } - if stdout.Len() != 0 { - t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr on success, got %q", stderr.String()) } - if !strings.Contains(stderr.String(), "runner_execution") { - t.Fatalf("expected runner_execution error, got %q", stderr.String()) + parsed, err := schema.ParseTranscriptJSON(stdout.Bytes()) + if err != nil { + t.Fatalf("expected transcript stdout on success: %v", err) + } + if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" { + t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments) } runDir := onlyRunDir(t, workDir) - if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { - t.Fatalf("expected error.log on failed glossary run: %v", err) + if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) { + t.Fatalf("did not expect error.log on successful glossary run: %v", err) } report := readProcessReport(t, reportPath) - if report.Status != "failed" || report.ErrorPhase != "runner_execution" { - t.Fatalf("expected failed runner_execution report, got %+v", report) + if report.Status != "success" || report.ErrorPhase != "" { + t.Fatalf("expected successful report, got %+v", report) + } + if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 { + t.Fatalf("expected one module warning, got %+v", report.ModuleResults) } } @@ -3295,7 +3313,7 @@ func TestRunProcessExplicitHomophonesProtectedGlossaryTermRejected(t *testing.T) } } -func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *testing.T) { +func TestRunProcessExplicitHomophonesMalformedLLMOutputSucceedsWithWarning(t *testing.T) { processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")} t.Cleanup(func() { processProposalLLMClient = nil }) @@ -3314,22 +3332,29 @@ func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *test "--work-dir-retention", "always", "--report-json", reportPath, }, &stdout, &stderr) - if exitCode == 0 { - t.Fatal("expected failure") + if exitCode != 0 { + t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) } - if stdout.Len() != 0 { - t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr on success, got %q", stderr.String()) } - if !strings.Contains(stderr.String(), "runner_execution") { - t.Fatalf("expected runner_execution error, got %q", stderr.String()) + parsed, err := schema.ParseTranscriptJSON(stdout.Bytes()) + if err != nil { + t.Fatalf("expected transcript stdout on success: %v", err) + } + if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" { + t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments) } runDir := onlyRunDir(t, workDir) - if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { - t.Fatalf("expected error.log on failed homophones run: %v", err) + if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) { + t.Fatalf("did not expect error.log on successful homophones run: %v", err) } report := readProcessReport(t, reportPath) - if report.Status != "failed" || report.ErrorPhase != "runner_execution" { - t.Fatalf("expected failed runner_execution report, got %+v", report) + if report.Status != "success" || report.ErrorPhase != "" { + t.Fatalf("expected successful report, got %+v", report) + } + if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 { + t.Fatalf("expected one module warning, got %+v", report.ModuleResults) } } @@ -3674,7 +3699,7 @@ func TestRunProcessExplicitSpokenWordProtectedGlossaryTermRejected(t *testing.T) } } -func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *testing.T) { +func TestRunProcessExplicitSpokenWordMalformedLLMOutputSucceedsWithWarning(t *testing.T) { processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")} t.Cleanup(func() { processProposalLLMClient = nil }) @@ -3693,22 +3718,29 @@ func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *test "--work-dir-retention", "always", "--report-json", reportPath, }, &stdout, &stderr) - if exitCode == 0 { - t.Fatal("expected failure") + if exitCode != 0 { + t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String()) } - if stdout.Len() != 0 { - t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr on success, got %q", stderr.String()) } - if !strings.Contains(stderr.String(), "runner_execution") { - t.Fatalf("expected runner_execution error, got %q", stderr.String()) + parsed, err := schema.ParseTranscriptJSON(stdout.Bytes()) + if err != nil { + t.Fatalf("expected transcript stdout on success: %v", err) + } + if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" { + t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments) } runDir := onlyRunDir(t, workDir) - if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { - t.Fatalf("expected error.log on failed spoken_word run: %v", err) + if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) { + t.Fatalf("did not expect error.log on successful spoken_word run: %v", err) } report := readProcessReport(t, reportPath) - if report.Status != "failed" || report.ErrorPhase != "runner_execution" { - t.Fatalf("expected failed runner_execution report, got %+v", report) + if report.Status != "success" || report.ErrorPhase != "" { + t.Fatalf("expected successful report, got %+v", report) + } + if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 { + t.Fatalf("expected one module warning, got %+v", report.ModuleResults) } } diff --git a/internal/cli/subprocess_test_hooks.go b/internal/cli/subprocess_test_hooks.go index d439bea..9ce18ec 100644 --- a/internal/cli/subprocess_test_hooks.go +++ b/internal/cli/subprocess_test_hooks.go @@ -78,25 +78,22 @@ func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req co } } case "mid_pipeline_fail": - switch target := out.(type) { - case *proposal_generation.StructuredCorrectionSet: + if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok { c.mu.Lock() c.proposals++ proposalCall := c.proposals c.mu.Unlock() if proposalCall >= 3 { - *target = proposal_generation.StructuredCorrectionSet{ - Corrections: []proposal_generation.StructuredCorrectionProposal{ - {TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99}, - }, - } - } else { - *target = proposal_generation.StructuredCorrectionSet{ - Corrections: []proposal_generation.StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99}, - }, - } + return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure") + } + } + switch target := out.(type) { + case *proposal_generation.StructuredCorrectionSet: + *target = proposal_generation.StructuredCorrectionSet{ + Corrections: []proposal_generation.StructuredCorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99}, + }, } case *validators.LLMValidationResponse: *target = validators.LLMValidationResponse{Validations: nil} diff --git a/internal/core/reporting/report.go b/internal/core/reporting/report.go index faf8244..86f1cd4 100644 --- a/internal/core/reporting/report.go +++ b/internal/core/reporting/report.go @@ -7,6 +7,7 @@ import ( "time" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) type ProcessReport struct { @@ -46,18 +47,19 @@ type ReportMetadata struct { } type ModuleReport struct { - ModuleKey string `json:"module_key"` - ModuleInstance string `json:"module_instance"` - ReplacementPolicy string `json:"replacement_policy,omitempty"` - Status string `json:"status"` - ProposalCount int `json:"proposal_count"` - ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"` - ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"` - AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` - SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - StartedAt *time.Time `json:"started_at,omitempty"` - CompletedAt *time.Time `json:"completed_at,omitempty"` + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + ReplacementPolicy string `json:"replacement_policy,omitempty"` + Status string `json:"status"` + ProposalCount int `json:"proposal_count"` + Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"` + ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"` + ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"` + AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` + SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` } type ValidatorDecisionReport struct { diff --git a/internal/framework/contracts/chunk_proposal_composition_test.go b/internal/framework/contracts/chunk_proposal_composition_test.go index 7d2d2c0..8cd2010 100644 --- a/internal/framework/contracts/chunk_proposal_composition_test.go +++ b/internal/framework/contracts/chunk_proposal_composition_test.go @@ -47,7 +47,7 @@ func (h testChunkProposalHarness) collectEnrichedProposals( return nil, err } - for _, proposal := range base { + for _, proposal := range base.Proposals { sectionIndex := section.Index out = append(out, proposals.EnrichedCorrectionProposal{ CorrectionProposal: proposal, @@ -85,11 +85,11 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy func (m deterministicFakeModule) Validators() []Validator { return nil } -func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) { _ = ctx if req.WorkingTranscript == nil || req.Section == nil { - return []proposals.CorrectionProposal{}, nil + return ProposalResult{}, nil } out := make([]proposals.CorrectionProposal, 0) @@ -110,7 +110,7 @@ func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalReques } } - return out, nil + return ProposalResult{Proposals: out}, nil } func TestChunkProposalMetadataAssociation(t *testing.T) { diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index 9fa4114..9a4ae52 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -11,6 +11,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" + stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) // StructuredLLMClient provides provider-agnostic structured completion. @@ -28,7 +29,7 @@ type TranscriptModule interface { Key() string ReplacementPolicy() proposals.ReplacementPolicy Validators() []Validator - Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) + Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) } // Validator evaluates candidate proposals and returns one decision per proposal index. @@ -103,6 +104,11 @@ type ProposalRequest struct { LLMScheduler LLMScheduler `json:"-"` } +type ProposalResult struct { + Proposals []proposals.CorrectionProposal `json:"proposals,omitempty"` + Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"` +} + // ValidationRequest is the input to validator execution. type ValidationRequest = validators.Request diff --git a/internal/framework/contracts/contracts_test.go b/internal/framework/contracts/contracts_test.go index 2a26d3e..7ee8adf 100644 --- a/internal/framework/contracts/contracts_test.go +++ b/internal/framework/contracts/contracts_test.go @@ -50,11 +50,13 @@ func (f *fakeModule) Validators() []Validator { return []Validator{&fakeValidator{}} } -func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) { _ = ctx _ = req - return []proposals.CorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9}, + return ProposalResult{ + Proposals: []proposals.CorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9}, + }, }, nil } @@ -68,12 +70,12 @@ func TestInterfaceContractsCompileWithFakes(t *testing.T) { t.Fatalf("unexpected module key: %q", got) } - proposalsOut, err := module.Propose(context.Background(), ProposalRequest{}) + proposalResult, err := module.Propose(context.Background(), ProposalRequest{}) if err != nil { t.Fatalf("unexpected propose error: %v", err) } - if len(proposalsOut) != 1 { - t.Fatalf("expected one proposal, got %d", len(proposalsOut)) + if len(proposalResult.Proposals) != 1 { + t.Fatalf("expected one proposal, got %d", len(proposalResult.Proposals)) } } diff --git a/internal/framework/modules/registry_test.go b/internal/framework/modules/registry_test.go index 3f88183..88b33c2 100644 --- a/internal/framework/modules/registry_test.go +++ b/internal/framework/modules/registry_test.go @@ -19,10 +19,10 @@ func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy { return proposals.ReplacementPolicyRequireUnique } func (m noopModule) Validators() []contracts.Validator { return nil } -func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { _ = ctx _ = req - return nil, nil + return contracts.ProposalResult{}, nil } func TestKnownModuleKeyRecognition(t *testing.T) { diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index a9cdacd..d00554e 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -15,6 +15,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" + stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) // InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts. @@ -68,6 +69,7 @@ type Request struct { type Result struct { Corrections []proposals.CorrectionProposal `json:"corrections"` Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"` + Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"` Artifacts InteractionArtifacts `json:"artifacts,omitempty"` } @@ -156,6 +158,12 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { } if callErr != nil { + if isMalformedStructuredOutputError(callErr) { + return Result{ + Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)}, + Artifacts: artifacts, + }, nil + } return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr) } @@ -168,9 +176,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { CorrectedText: raw.CorrectedText, Confidence: raw.Confidence, } - if err := candidate.Validate(); err != nil { - return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err) - } corrections = append(corrections, candidate) enrichedCandidate := proposals.EnrichedCorrectionProposal{ @@ -191,6 +196,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { return Result{ Corrections: corrections, Enriched: enriched, + Warnings: nil, Artifacts: artifacts, }, nil } @@ -240,6 +246,48 @@ func errPayload(err error) any { return map[string]any{"error": err.Error()} } +func isMalformedStructuredOutputError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + for _, marker := range []string{ + "malformed structured output", + "decode structured output:", + "decode provider response envelope:", + "provider response missing choices", + "provider response missing assistant message content", + "provider response assistant message content is empty", + "provider response assistant message content is not valid JSON", + } { + if strings.Contains(msg, marker) { + return true + } + } + return false +} + +func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning { + warning := stagewarnings.StageWarning{ + Scope: stagewarnings.ScopeProposalGeneration, + ReasonCode: "proposal_response_malformed", + Message: strings.TrimSpace(err.Error()), + DiagnosticArtifactPath: diagnosticArtifactPath(artifacts), + } + if section != nil { + sectionIndex := section.Index + warning.SectionIndex = §ionIndex + } + return warning +} + +func diagnosticArtifactPath(artifacts InteractionArtifacts) string { + if artifacts.ErrorPayloadPath != "" { + return artifacts.ErrorPayloadPath + } + return artifacts.ResponsePayloadPath +} + type diagnosticsWriterAdapter struct { writer *llm.DiagnosticsWriter } diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index 9e05fca..0cc53ce 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -224,12 +224,12 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) { } } -func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) { +func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ { Corrections: []StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9}, + {TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2}, }, }, }, @@ -237,9 +237,38 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) { req := defaultRequest(t) req.LLMClient = client - _, err := GenerateCandidates(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "invalid structured correction") { - t.Fatalf("expected structured response validation failure, got %v", err) + result, err := GenerateCandidates(context.Background(), req) + if err != nil { + t.Fatalf("expected invalid correction to survive generation, got %v", err) + } + if len(result.Corrections) != 1 { + t.Fatalf("expected one correction, got %+v", result) + } + if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 { + t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0]) + } + if len(result.Warnings) != 0 { + t.Fatalf("did not expect warnings for individually invalid corrections, got %+v", result.Warnings) + } +} + +func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) { + client := &fakeStructuredClient{err: errors.New("malformed structured output")} + req := defaultRequest(t) + req.LLMClient = client + + result, err := GenerateCandidates(context.Background(), req) + if err != nil { + t.Fatalf("expected malformed structured output to downgrade to warning, got %v", err) + } + if len(result.Corrections) != 0 || len(result.Enriched) != 0 { + t.Fatalf("expected no proposals on malformed response, got %+v", result) + } + if len(result.Warnings) != 1 { + t.Fatalf("expected one warning, got %+v", result.Warnings) + } + if result.Warnings[0].ReasonCode != "proposal_response_malformed" { + t.Fatalf("unexpected warning: %+v", result.Warnings[0]) } } diff --git a/internal/framework/proposals/apply.go b/internal/framework/proposals/apply.go index 4c66cea..b9cef9f 100644 --- a/internal/framework/proposals/apply.go +++ b/internal/framework/proposals/apply.go @@ -147,6 +147,8 @@ func skipReasonMessage(reason ProposalSkipReason) string { return "original_text matched multiple spans under require_unique policy" case SkipReasonNoEffect: return "original_text and corrected_text are identical" + case SkipReasonEmptyResultingText: + return "proposal would leave the segment empty" case SkipReasonInvalidProposal: return "proposal failed structural or policy validation" default: diff --git a/internal/framework/proposals/preview.go b/internal/framework/proposals/preview.go index ccc5c21..b81e459 100644 --- a/internal/framework/proposals/preview.go +++ b/internal/framework/proposals/preview.go @@ -15,6 +15,7 @@ const ( SkipReasonMissingOriginalText ProposalSkipReason = "missing_original_text" SkipReasonAmbiguousOriginal ProposalSkipReason = "ambiguous_original_text" SkipReasonNoEffect ProposalSkipReason = "no_effect" + SkipReasonEmptyResultingText ProposalSkipReason = "empty_resulting_segment" SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal" ) @@ -62,6 +63,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo } corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1) + if strings.TrimSpace(corrected) == "" { + return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText} + } return SegmentPreviewResult{ Applicable: true, CorrectedSegmentText: corrected, @@ -70,6 +74,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo case ReplacementPolicyReplaceAll: corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText) + if strings.TrimSpace(corrected) == "" { + return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText} + } return SegmentPreviewResult{ Applicable: true, CorrectedSegmentText: corrected, diff --git a/internal/framework/proposals/preview_test.go b/internal/framework/proposals/preview_test.go index 39564d0..2e8aabe 100644 --- a/internal/framework/proposals/preview_test.go +++ b/internal/framework/proposals/preview_test.go @@ -121,6 +121,42 @@ func TestPreviewProposalForSegmentNoEffectReplacement(t *testing.T) { } } +func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonEmpty(t *testing.T) { + segment := &schema.Segment{ID: 8, Text: "uh hello"} + proposal := CorrectionProposal{ + TargetSegmentID: 8, + OriginalText: "uh ", + CorrectedText: "", + Confidence: 0.9, + } + + result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique) + if !result.Applicable { + t.Fatalf("expected applicable preview, got skip reason %q", result.SkipReason) + } + if result.CorrectedSegmentText != "hello" { + t.Fatalf("unexpected corrected text: %q", result.CorrectedSegmentText) + } +} + +func TestPreviewProposalForSegmentRejectsEmptyResultingSegment(t *testing.T) { + segment := &schema.Segment{ID: 8, Text: "uh"} + proposal := CorrectionProposal{ + TargetSegmentID: 8, + OriginalText: "uh", + CorrectedText: "", + Confidence: 0.9, + } + + result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique) + if result.Applicable { + t.Fatal("expected non-applicable preview") + } + if result.SkipReason != SkipReasonEmptyResultingText { + t.Fatalf("expected skip reason %q, got %q", SkipReasonEmptyResultingText, result.SkipReason) + } +} + func TestPreviewProposalForSegmentPreservesInputSegment(t *testing.T) { segment := &schema.Segment{ID: 9, Speaker: "A", Start: 1.0, End: 2.0, Text: "rank rank"} original := *segment diff --git a/internal/framework/proposals/proposal.go b/internal/framework/proposals/proposal.go index a55e1b0..8dd2a10 100644 --- a/internal/framework/proposals/proposal.go +++ b/internal/framework/proposals/proposal.go @@ -38,9 +38,6 @@ func (p CorrectionProposal) Validate() error { if strings.TrimSpace(p.OriginalText) == "" { return fmt.Errorf("proposal original_text must not be empty") } - if strings.TrimSpace(p.CorrectedText) == "" { - return fmt.Errorf("proposal corrected_text must not be empty") - } if p.Confidence < 0.0 || p.Confidence > 1.0 { return fmt.Errorf("proposal confidence must be between 0.0 and 1.0") } diff --git a/internal/framework/proposals/proposal_test.go b/internal/framework/proposals/proposal_test.go index 93e27b8..d267521 100644 --- a/internal/framework/proposals/proposal_test.go +++ b/internal/framework/proposals/proposal_test.go @@ -35,7 +35,7 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) { } } -func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) { +func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) { proposal := CorrectionProposal{ TargetSegmentID: 42, OriginalText: "gestures", @@ -43,12 +43,8 @@ func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) { Confidence: 0.95, } - err := proposal.Validate() - if err == nil { - t.Fatal("expected validation error, got nil") - } - if err.Error() != "proposal corrected_text must not be empty" { - t.Fatalf("unexpected error: %v", err) + if err := proposal.Validate(); err != nil { + t.Fatalf("expected empty corrected_text to be allowed, got %v", err) } } diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index e33c608..e787e4e 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -15,6 +15,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" + stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata" ) @@ -37,18 +38,19 @@ type ValidationScheduler = contracts.LLMScheduler // ModuleResult captures deterministic per-module execution output. type ModuleResult struct { - ModuleKey string `json:"module_key"` - ModuleInstance string `json:"module_instance"` - ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` - Status string `json:"status"` - ProposalCount int `json:"proposal_count"` - ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"` - ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"` - AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` - SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - StartedAt time.Time `json:"started_at"` - CompletedAt time.Time `json:"completed_at"` + ModuleKey string `json:"module_key"` + ModuleInstance string `json:"module_instance"` + ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` + Status string `json:"status"` + ProposalCount int `json:"proposal_count"` + Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"` + ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"` + ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"` + AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` + SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` } type ValidatorDecisionRecord struct { @@ -176,6 +178,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { ReplacementPolicy: policy, Status: ModuleStatusFailed, ProposalCount: pipelineResult.ProposalCount, + Warnings: pipelineResult.Warnings, ValidatorDecisions: pipelineResult.ValidatorDecisions, ValidatorRejected: pipelineResult.ValidatorRejected, ErrorMessage: pipelineErr.Error(), @@ -199,6 +202,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) { ReplacementPolicy: policy, Status: ModuleStatusSuccess, ProposalCount: pipelineResult.ProposalCount, + Warnings: pipelineResult.Warnings, ValidatorDecisions: pipelineResult.ValidatorDecisions, ValidatorRejected: pipelineResult.ValidatorRejected, AppliedChanges: applyResult.Applied, @@ -232,12 +236,14 @@ type collectSectionProposalsInput struct { type sectionProposals struct { meta contracts.SectionMetadata corrected []proposals.CorrectionProposal + warnings []stagewarnings.StageWarning } type sectionProposalResult struct { sectionPos int section chunking.Section corrected []proposals.CorrectionProposal + warnings []stagewarnings.StageWarning err error } @@ -247,12 +253,14 @@ type sectionValidationResult struct { approved []proposals.EnrichedCorrectionProposal decisions []ValidatorDecisionRecord rejected []ValidatorRejectedChange + warnings []stagewarnings.StageWarning err error } type modulePipelineResult struct { ProposalCount int Approved []proposals.EnrichedCorrectionProposal + Warnings []stagewarnings.StageWarning ValidatorDecisions []ValidatorDecisionRecord ValidatorRejected []ValidatorRejectedChange } @@ -271,7 +279,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI go func() { defer wg.Done() meta := contracts.SectionMetadataFromSection(section) - corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{ + proposalResult, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ Config: input.Config, WorkingTranscript: transcriptFromSection(section), @@ -291,7 +299,8 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI case results <- sectionProposalResult{ sectionPos: sectionPos, section: section, - corrected: corrected, + corrected: proposalResult.Proposals, + warnings: proposalResult.Warnings, err: err, }: case <-runCtx.Done(): @@ -308,6 +317,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) { out := modulePipelineResult{ Approved: make([]proposals.EnrichedCorrectionProposal, 0), + Warnings: make([]stagewarnings.StageWarning, 0), ValidatorDecisions: make([]ValidatorDecisionRecord, 0), ValidatorRejected: make([]ValidatorRejectedChange, 0), } @@ -352,6 +362,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) if firstErr != nil { continue } + out.Warnings = append(out.Warnings, result.warnings...) pending[result.sectionPos] = result for { @@ -401,6 +412,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) approved: validated.approved, decisions: validated.decisions, rejected: validated.rejected, + warnings: validated.warnings, err: err, } }(nextSectionToProcess, sectionEnriched, sectionMeta) @@ -424,6 +436,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) break } out.Approved = append(out.Approved, res.approved...) + out.Warnings = append(out.Warnings, res.warnings...) out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...) out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...) } @@ -440,6 +453,38 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) } return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName] }) + sort.SliceStable(out.Warnings, func(i, j int) bool { + leftSection, rightSection := -1, -1 + if out.Warnings[i].SectionIndex != nil { + leftSection = *out.Warnings[i].SectionIndex + } + if out.Warnings[j].SectionIndex != nil { + rightSection = *out.Warnings[j].SectionIndex + } + if leftSection != rightSection { + return leftSection < rightSection + } + leftBatch, rightBatch := -1, -1 + if out.Warnings[i].BatchIndex != nil { + leftBatch = *out.Warnings[i].BatchIndex + } + if out.Warnings[j].BatchIndex != nil { + rightBatch = *out.Warnings[j].BatchIndex + } + if leftBatch != rightBatch { + return leftBatch < rightBatch + } + if out.Warnings[i].ValidatorName != out.Warnings[j].ValidatorName { + return validatorOrder[out.Warnings[i].ValidatorName] < validatorOrder[out.Warnings[j].ValidatorName] + } + if out.Warnings[i].Scope != out.Warnings[j].Scope { + return out.Warnings[i].Scope < out.Warnings[j].Scope + } + if out.Warnings[i].ReasonCode != out.Warnings[j].ReasonCode { + return out.Warnings[i].ReasonCode < out.Warnings[j].ReasonCode + } + return out.Warnings[i].Message < out.Warnings[j].Message + }) if firstErr != nil { return out, firstErr @@ -467,11 +512,13 @@ type validateSectionCandidatesResult struct { approved []proposals.EnrichedCorrectionProposal decisions []ValidatorDecisionRecord rejected []ValidatorRejectedChange + warnings []stagewarnings.StageWarning } func validateSectionCandidates(ctx context.Context, input validateSectionCandidatesInput) (validateSectionCandidatesResult, error) { decisions := make([]ValidatorDecisionRecord, 0) rejected := make([]ValidatorRejectedChange, 0) + warnings := make([]stagewarnings.StageWarning, 0) eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...) for _, validator := range input.Validators { @@ -512,6 +559,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida approved: eligible, decisions: decisions, rejected: rejected, + warnings: warnings, }, fmt.Errorf("validator %q failed: %w", validator.Name(), err) } if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil { @@ -519,8 +567,10 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida approved: eligible, decisions: decisions, rejected: rejected, + warnings: warnings, }, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err) } + warnings = append(warnings, vResult.Warnings...) nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible)) byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible)) @@ -562,6 +612,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida approved: eligible, decisions: decisions, rejected: rejected, + warnings: warnings, }, nil } diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 09eae98..5d3948d 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -47,11 +47,12 @@ type fakeModule struct { func (m fakeModule) Key() string { return m.key } func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m fakeModule) Validators() []contracts.Validator { return m.validators } -func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { if m.proposeF == nil { - return nil, nil + return contracts.ProposalResult{}, nil } - return m.proposeF(req) + proposalsOut, err := m.proposeF(req) + return contracts.ProposalResult{Proposals: proposalsOut}, err } type fakeValidator struct { @@ -1191,7 +1192,7 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { } } -func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) { +func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ @@ -1209,15 +1210,21 @@ func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing. ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}, ValidationLLMClient: client, }) - if err == nil { - t.Fatal("expected llm validator failure") + if err != nil { + t.Fatalf("expected malformed validator response to downgrade, got %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected partial progress retained") } + if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 { + t.Fatalf("expected second module rejection, got %+v", out.ModuleResults) + } + if len(out.ModuleResults[1].Warnings) != 1 || out.ModuleResults[1].Warnings[0].ReasonCode != validators.ReasonValidatorMalformed { + t.Fatalf("expected malformed warning, got %+v", out.ModuleResults[1].Warnings) + } } -func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) { +func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ @@ -1232,12 +1239,12 @@ func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) { ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) - if err == nil { - t.Fatal("expected missing decision failure") + if err != nil { + t.Fatalf("expected missing decision downgrade, got %v", err) } } -func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) { +func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, @@ -1255,8 +1262,8 @@ func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) { ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) - if err == nil { - t.Fatal("expected duplicate decision failure") + if err != nil { + t.Fatalf("expected duplicate decision downgrade, got %v", err) } } @@ -1355,7 +1362,7 @@ type proposalGenerationModule struct { func (m proposalGenerationModule) Key() string { return m.key } func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m proposalGenerationModule) Validators() []contracts.Validator { return nil } -func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ ModuleKey: req.RunSpec.ModuleKey, ModuleInstance: req.RunSpec.InstanceName, @@ -1372,9 +1379,9 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { - return nil, err + return contracts.ProposalResult{}, err } - return result.Corrections, nil + return contracts.ProposalResult{Proposals: result.Corrections, Warnings: result.Warnings}, nil } type fakeProposalStructuredClient struct { diff --git a/internal/framework/validators/deterministic.go b/internal/framework/validators/deterministic.go index 81fda80..8fc058c 100644 --- a/internal/framework/validators/deterministic.go +++ b/internal/framework/validators/deterministic.go @@ -4,8 +4,35 @@ import ( "context" "fmt" "strings" + + "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" ) +type ProposalShapeValidator struct{} + +func (v ProposalShapeValidator) Name() string { return "proposal_shape" } + +func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result, error) { + decisions := make([]Decision, 0, len(req.CandidateProposal)) + for _, c := range req.CandidateProposal { + switch { + case c.TargetSegmentID <= 0: + decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive")) + case strings.TrimSpace(c.OriginalText) == "": + decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty")) + case c.Confidence < 0.0 || c.Confidence > 1.0: + decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0")) + default: + decisions = append(decisions, approval(c.ProposalIndex)) + } + } + if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil { + return Result{}, err + } + return Result{ValidatorName: v.Name(), Decisions: decisions}, nil +} + type ConfidenceThresholdValidator struct{} func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" } @@ -62,10 +89,23 @@ type NonEmptyCorrectionValidator struct{} func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" } func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) { + segmentsByID := make(map[int]schema.Segment) + if req.WorkingTranscript != nil { + for _, seg := range req.WorkingTranscript.Segments { + segmentsByID[seg.ID] = seg + } + } + decisions := make([]Decision, 0, len(req.CandidateProposal)) for _, c := range req.CandidateProposal { - if strings.TrimSpace(c.CorrectedText) == "" { - decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty")) + segment, ok := segmentsByID[c.TargetSegmentID] + if !ok { + decisions = append(decisions, approval(c.ProposalIndex)) + continue + } + preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy) + if preview.SkipReason == proposals.SkipReasonEmptyResultingText { + decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty")) continue } decisions = append(decisions, approval(c.ProposalIndex)) diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 634d709..27cd475 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -11,6 +11,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" + stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" "gitea.maximumdirect.net/eric/audita/internal/prompts" ) @@ -78,7 +79,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, maxTokens = req.Config.ValidationMaxPromptTokens } - batches, err := ChunkLLMValidationItems(validationReq.Items, maxTokens, v.estimator) + warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...) + oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator) + itemsForBatching := filterItemsByDecision(validationReq.Items, oversized) + if len(itemsForBatching) == 0 { + all := append([]Decision(nil), immediate...) + all = append(all, oversized...) + if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { + return Result{}, err + } + sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) + return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil + } + + batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator) if err != nil { return Result{}, err } @@ -140,12 +154,19 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, ) } if err != nil { + if isMalformedStructuredOutputError(err) { + llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...) + warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts)) + continue + } return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) } batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response) if err != nil { - return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err) + llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...) + warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts)) + continue } for i := range batchDecisions { batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath @@ -154,12 +175,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } all := append([]Decision(nil), immediate...) + all = append(all, oversized...) all = append(all, llmDecisions...) if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { return Result{}, err } sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) - return Result{ValidatorName: v.name, Decisions: all}, nil + return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil } func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata { @@ -301,3 +323,98 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation } return decisions, nil } + +func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision { + out := make([]Decision, 0) + for _, item := range items { + singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item}) + if err != nil || singleTokens <= maxPromptTokens { + continue + } + out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens")) + } + return out +} + +func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning { + out := make([]stagewarnings.StageWarning, 0) + for _, item := range items { + singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item}) + if err != nil || singleTokens <= maxPromptTokens { + continue + } + out = append(out, stagewarnings.StageWarning{ + Scope: stagewarnings.ScopeValidator, + ValidatorName: validatorName, + ReasonCode: ReasonValidatorInputTooLarge, + Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex), + }) + } + return out +} + +func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem { + if len(decisions) == 0 { + return append([]LLMValidationItem(nil), items...) + } + rejected := make(map[int]struct{}, len(decisions)) + for _, decision := range decisions { + rejected[decision.ProposalIndex] = struct{}{} + } + out := make([]LLMValidationItem, 0, len(items)) + for _, item := range items { + if _, ok := rejected[item.CorrectionIndex]; ok { + continue + } + out = append(out, item) + } + return out +} + +func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision { + out := make([]Decision, 0, len(items)) + for _, item := range items { + out = append(out, rejection(item.CorrectionIndex, reasonCode, message)) + } + return out +} + +func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning { + idx := batchIndex + return stagewarnings.StageWarning{ + Scope: stagewarnings.ScopeValidator, + ValidatorName: validatorName, + BatchIndex: &idx, + ReasonCode: reasonCode, + Message: strings.TrimSpace(message), + DiagnosticArtifactPath: diagnosticArtifactPath(artifacts), + } +} + +func diagnosticArtifactPath(artifacts InteractionArtifacts) string { + if artifacts.ErrorPayloadPath != "" { + return artifacts.ErrorPayloadPath + } + return artifacts.ResponsePayloadPath +} + +func isMalformedStructuredOutputError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + for _, marker := range []string{ + "malformed structured output", + "decode structured output:", + "decode provider response envelope:", + "provider response missing choices", + "provider response missing assistant message content", + "provider response assistant message content is empty", + "provider response assistant message content is not valid JSON", + } { + if strings.Contains(msg, marker) { + return true + } + } + return false +} diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index 6830e2a..df98712 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -248,29 +248,38 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) { } } -func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) { +func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) { client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")} v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - _, err := v.Validate(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "completion failed") { - t.Fatalf("expected malformed output error, got %v", err) + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("expected malformed output downgrade, got %v", err) + } + if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + t.Fatalf("unexpected decisions: %+v", res.Decisions) + } + if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed { + t.Fatalf("expected malformed warning, got %+v", res.Warnings) } } -func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) { +func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}} v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - _, err := v.Validate(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "response invalid") { - t.Fatalf("expected missing decision error, got %v", err) + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("expected missing decision downgrade, got %v", err) + } + if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + t.Fatalf("unexpected decisions: %+v", res.Decisions) } } -func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) { +func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, @@ -278,20 +287,74 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) { v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - _, err := v.Validate(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "duplicate") { - t.Fatalf("expected duplicate decision error, got %v", err) + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("expected duplicate decision downgrade, got %v", err) + } + if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + t.Fatalf("unexpected decisions: %+v", res.Decisions) } } -func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) { +func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}} v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client - _, err := v.Validate(context.Background(), req) - if err == nil || !strings.Contains(err.Error(), "unknown") { - t.Fatalf("expected unknown index error, got %v", err) + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("expected unknown index downgrade, got %v", err) + } + if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + t.Fatalf("unexpected decisions: %+v", res.Decisions) + } +} + +func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) { + client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ + {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}, + }}}} + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + huge := strings.Repeat("gestures ", 200) + req := Request{ + WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{ + {ID: 1, Text: huge}, + {ID: 2, Text: "There were gestures at the temple.", Categories: []string{"narration"}}, + }}, + CandidateProposal: []proposals.EnrichedCorrectionProposal{ + { + CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9}, + ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"}, + }, + { + CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9}, + ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"}, + }, + }, + ModuleKey: "homophones", + ModuleInstance: "homophones", + ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, + } + req.LLMClient = client + cfg := config.Default() + cfg.ValidationMaxPromptTokens = 200 + req.Config = &cfg + + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("expected oversize downgrade, got %v", err) + } + if len(res.Decisions) != 2 { + t.Fatalf("expected two decisions, got %+v", res.Decisions) + } + if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved { + t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0]) + } + if !res.Decisions[1].Approved { + t.Fatalf("expected second decision approved, got %+v", res.Decisions[1]) + } + if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge { + t.Fatalf("expected one oversize warning, got %+v", res.Warnings) } } diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 0aae19f..c1f4f6d 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -8,16 +8,22 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) const ( - ReasonApproved = "approved" - ReasonLowConfidence = "low_confidence" - ReasonMissingOriginalText = "missing_original_text" - ReasonMissingTargetSegment = "missing_target_segment" - ReasonEmptyCorrectedText = "empty_corrected_text" - ReasonNoEffect = "no_effect" - ReasonProtectedGlossaryTerm = "protected_glossary_term" + ReasonApproved = "approved" + ReasonLowConfidence = "low_confidence" + ReasonMissingOriginalText = "missing_original_text" + ReasonMissingTargetSegment = "missing_target_segment" + ReasonEmptyResultingText = "empty_resulting_segment" + ReasonNoEffect = "no_effect" + ReasonProtectedGlossaryTerm = "protected_glossary_term" + ReasonInvalidTargetSegment = "invalid_target_segment_id" + ReasonEmptyOriginalText = "empty_original_text" + ReasonInvalidConfidence = "invalid_confidence" + ReasonValidatorMalformed = "validator_response_malformed" + ReasonValidatorInputTooLarge = "validator_input_too_large" ) // Request is the runtime input shared by deterministic validators. @@ -45,8 +51,9 @@ type Decision struct { // Result is one validator output containing exactly one decision per proposal index. type Result struct { - ValidatorName string `json:"validator_name"` - Decisions []Decision `json:"decisions"` + ValidatorName string `json:"validator_name"` + Decisions []Decision `json:"decisions"` + Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"` } // ValidationScheduler provides bounded execution for validator LLM calls. diff --git a/internal/framework/validators/validators_test.go b/internal/framework/validators/validators_test.go index ba18a05..0600cc3 100644 --- a/internal/framework/validators/validators_test.go +++ b/internal/framework/validators/validators_test.go @@ -68,6 +68,31 @@ func TestConfidenceThresholdValidator(t *testing.T) { } } +func TestProposalShapeValidator(t *testing.T) { + req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{ + mkCandidate(0, 1, "teh", "the", 0.9), + mkCandidate(1, 0, "teh", "the", 0.9), + mkCandidate(2, 1, " ", "the", 0.9), + mkCandidate(3, 1, "teh", "the", 1.5), + }} + res, err := (ProposalShapeValidator{}).Validate(context.Background(), req) + if err != nil { + t.Fatalf("Validate error: %v", err) + } + if !res.Decisions[0].Approved { + t.Fatalf("expected proposal 0 approved") + } + if res.Decisions[1].ReasonCode != ReasonInvalidTargetSegment { + t.Fatalf("expected invalid target segment rejection, got %+v", res.Decisions[1]) + } + if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText { + t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2]) + } + if res.Decisions[3].ReasonCode != ReasonInvalidConfidence { + t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3]) + } +} + func TestOriginalTextPresenceValidator(t *testing.T) { req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{ mkCandidate(0, 1, "hello", "hi", 0.9), @@ -90,10 +115,13 @@ func TestOriginalTextPresenceValidator(t *testing.T) { } func TestNonEmptyCorrectionValidator(t *testing.T) { - req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{ - mkCandidate(0, 1, "hello", "hi", 0.9), - mkCandidate(1, 1, "hello", " ", 0.9), - }} + req := Request{ + WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, + ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, + CandidateProposal: []proposals.EnrichedCorrectionProposal{ + mkCandidate(0, 1, "hello", "hi", 0.9), + mkCandidate(1, 1, "hello world", " ", 0.9), + }} res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req) if err != nil { t.Fatalf("Validate error: %v", err) @@ -101,8 +129,8 @@ func TestNonEmptyCorrectionValidator(t *testing.T) { if !res.Decisions[0].Approved { t.Fatalf("expected proposal 0 approved") } - if res.Decisions[1].ReasonCode != ReasonEmptyCorrectedText { - t.Fatalf("expected empty_corrected_text, got %+v", res.Decisions[1]) + if res.Decisions[1].ReasonCode != ReasonEmptyResultingText { + t.Fatalf("expected empty_resulting_segment, got %+v", res.Decisions[1]) } } @@ -151,9 +179,14 @@ func TestStableReasonCodes(t *testing.T) { ReasonLowConfidence, ReasonMissingOriginalText, ReasonMissingTargetSegment, - ReasonEmptyCorrectedText, + ReasonEmptyResultingText, ReasonNoEffect, ReasonProtectedGlossaryTerm, + ReasonInvalidTargetSegment, + ReasonEmptyOriginalText, + ReasonInvalidConfidence, + ReasonValidatorMalformed, + ReasonValidatorInputTooLarge, } for _, code := range codes { if strings.TrimSpace(code) == "" { diff --git a/internal/framework/warnings/warnings.go b/internal/framework/warnings/warnings.go new file mode 100644 index 0000000..8f75478 --- /dev/null +++ b/internal/framework/warnings/warnings.go @@ -0,0 +1,18 @@ +package warnings + +type Scope string + +const ( + ScopeProposalGeneration Scope = "proposal_generation" + ScopeValidator Scope = "validator" +) + +type StageWarning struct { + Scope Scope `json:"scope"` + ReasonCode string `json:"reason_code"` + Message string `json:"message"` + SectionIndex *int `json:"section_index,omitempty"` + ValidatorName string `json:"validator_name,omitempty"` + BatchIndex *int `json:"batch_index,omitempty"` + DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` +} diff --git a/internal/modules/glossary/module.go b/internal/modules/glossary/module.go index 2448b86..4da5c1b 100644 --- a/internal/modules/glossary/module.go +++ b/internal/modules/glossary/module.go @@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator { return append([]contracts.Validator(nil), m.validators...) } -func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section) sectionIndex := 0 if req.Section != nil { @@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] } messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription) if err != nil { - return nil, err + return contracts.ProposalResult{}, err } generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ @@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { - return nil, err + return contracts.ProposalResult{}, err } - return generated.Corrections, nil + return contracts.ProposalResult{ + Proposals: generated.Corrections, + Warnings: generated.Warnings, + }, nil } func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript { diff --git a/internal/modules/glossary/module_test.go b/internal/modules/glossary/module_test.go index d832a3f..122379d 100644 --- a/internal/modules/glossary/module_test.go +++ b/internal/modules/glossary/module_test.go @@ -129,6 +129,7 @@ func TestGlossaryModuleValidatorChain(t *testing.T) { got = append(got, v.Name()) } want := []string{ + "proposal_shape", "no_effect", "original_text_presence", "confidence_threshold", @@ -185,7 +186,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) if len(client.calls) != 1 || client.calls[0].StageName != "glossary:proposal" { t.Fatalf("expected one glossary:proposal call, got %+v", client.calls) } - if len(out) != 1 || out[0].CorrectedText != "Jesters" { + if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" { t.Fatalf("unexpected proposals: %+v", out) } diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "glossary", "*proposal*response-payload.json")) diff --git a/internal/modules/grammar/module.go b/internal/modules/grammar/module.go index 6bdefc3..8a5d97f 100644 --- a/internal/modules/grammar/module.go +++ b/internal/modules/grammar/module.go @@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator { return append([]contracts.Validator(nil), m.validators...) } -func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section) sectionIndex := 0 if req.Section != nil { @@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] } messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription) if err != nil { - return nil, err + return contracts.ProposalResult{}, err } generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ @@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { - return nil, err + return contracts.ProposalResult{}, err } - return generated.Corrections, nil + return contracts.ProposalResult{ + Proposals: generated.Corrections, + Warnings: generated.Warnings, + }, nil } func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript { diff --git a/internal/modules/grammar/module_test.go b/internal/modules/grammar/module_test.go index cd9bf15..a793aca 100644 --- a/internal/modules/grammar/module_test.go +++ b/internal/modules/grammar/module_test.go @@ -131,6 +131,7 @@ func TestGrammarModuleValidatorChain(t *testing.T) { got = append(got, v.Name()) } want := []string{ + "proposal_shape", "no_effect", "original_text_presence", "confidence_threshold", @@ -187,7 +188,7 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) { if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal" { t.Fatalf("expected one grammar:proposal call, got %+v", client.calls) } - if len(out) != 1 || out[0].CorrectedText != "Hello, world" { + if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Hello, world" { t.Fatalf("unexpected proposals: %+v", out) } diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "grammar", "*proposal*response-payload.json")) diff --git a/internal/modules/homophones/module.go b/internal/modules/homophones/module.go index e07dbdd..4e3a068 100644 --- a/internal/modules/homophones/module.go +++ b/internal/modules/homophones/module.go @@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator { return append([]contracts.Validator(nil), m.validators...) } -func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section) sectionIndex := 0 if req.Section != nil { @@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] } messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription) if err != nil { - return nil, err + return contracts.ProposalResult{}, err } generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ @@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { - return nil, err + return contracts.ProposalResult{}, err } - return generated.Corrections, nil + return contracts.ProposalResult{ + Proposals: generated.Corrections, + Warnings: generated.Warnings, + }, nil } func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript { diff --git a/internal/modules/homophones/module_test.go b/internal/modules/homophones/module_test.go index 3e795c9..e79da74 100644 --- a/internal/modules/homophones/module_test.go +++ b/internal/modules/homophones/module_test.go @@ -145,6 +145,7 @@ func TestHomophonesModuleValidatorChain(t *testing.T) { got = append(got, v.Name()) } want := []string{ + "proposal_shape", "no_effect", "original_text_presence", "confidence_threshold", @@ -201,7 +202,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" { t.Fatalf("expected one homophones:proposal call, got %+v", client.calls) } - if len(out) != 1 || out[0].CorrectedText != "Jesters" { + if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" { t.Fatalf("unexpected proposals: %+v", out) } diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json")) diff --git a/internal/modules/spoken_word/module.go b/internal/modules/spoken_word/module.go index 1b8bed0..60261af 100644 --- a/internal/modules/spoken_word/module.go +++ b/internal/modules/spoken_word/module.go @@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator { return append([]contracts.Validator(nil), m.validators...) } -func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { +func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section) sectionIndex := 0 if req.Section != nil { @@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] } messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription) if err != nil { - return nil, err + return contracts.ProposalResult{}, err } generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ @@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([] DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { - return nil, err + return contracts.ProposalResult{}, err } - return generated.Corrections, nil + return contracts.ProposalResult{ + Proposals: generated.Corrections, + Warnings: generated.Warnings, + }, nil } func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript { diff --git a/internal/modules/spoken_word/module_test.go b/internal/modules/spoken_word/module_test.go index 05f7779..37c25e1 100644 --- a/internal/modules/spoken_word/module_test.go +++ b/internal/modules/spoken_word/module_test.go @@ -132,6 +132,7 @@ func TestSpokenWordModuleValidatorChain(t *testing.T) { got = append(got, v.Name()) } want := []string{ + "proposal_shape", "no_effect", "original_text_presence", "confidence_threshold", @@ -188,7 +189,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T if len(client.calls) != 1 || client.calls[0].StageName != "spoken_word:proposal" { t.Fatalf("expected one spoken_word:proposal call, got %+v", client.calls) } - if len(out) != 1 || out[0].CorrectedText != "I think" { + if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "I think" { t.Fatalf("unexpected proposals: %+v", out) } diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "spoken_word", "*proposal*response-payload.json")) diff --git a/internal/validators/chains.go b/internal/validators/chains.go index 5513fd2..eec7549 100644 --- a/internal/validators/chains.go +++ b/internal/validators/chains.go @@ -9,6 +9,7 @@ import ( var builtInChains = map[string][]string{ "glossary": { + KeyProposalShape, KeyNoEffect, KeyOriginalTextPresence, KeyConfidenceThreshold, @@ -18,6 +19,7 @@ var builtInChains = map[string][]string{ KeyMeaningReversalReview, }, "homophones": { + KeyProposalShape, KeyNoEffect, KeyOriginalTextPresence, KeyConfidenceThreshold, @@ -27,6 +29,7 @@ var builtInChains = map[string][]string{ KeyMeaningReversalReview, }, "spoken_word": { + KeyProposalShape, KeyNoEffect, KeyOriginalTextPresence, KeyConfidenceThreshold, @@ -36,6 +39,7 @@ var builtInChains = map[string][]string{ KeyMeaningReversalReview, }, "grammar": { + KeyProposalShape, KeyNoEffect, KeyOriginalTextPresence, KeyConfidenceThreshold, diff --git a/internal/validators/proposal_shape/validator.go b/internal/validators/proposal_shape/validator.go new file mode 100644 index 0000000..61911f1 --- /dev/null +++ b/internal/validators/proposal_shape/validator.go @@ -0,0 +1,13 @@ +package proposal_shape + +import ( + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators" + validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata" +) + +const Key = "proposal_shape" + +func New() (contracts.Validator, error) { + return validatormetadata.Wrap(frameworkvalidators.ProposalShapeValidator{}, validatormetadata.ExecutionClassDeterministic), nil +} diff --git a/internal/validators/registry.go b/internal/validators/registry.go index 6bd5b94..4763319 100644 --- a/internal/validators/registry.go +++ b/internal/validators/registry.go @@ -11,11 +11,13 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/validators/no_effect" "gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text" "gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence" + "gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape" "gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms" "gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility" ) const ( + KeyProposalShape = "proposal_shape" KeyConfidenceThreshold = "confidence_threshold" KeyOriginalTextPresence = "original_text_presence" KeyNonEmptyCorrectedText = "non_empty_corrected_text" @@ -39,6 +41,7 @@ type Registry struct { func NewBuiltInRegistry() *Registry { defs := []BuiltInValidatorDefinition{ + {Key: KeyProposalShape, Build: proposal_shape.New}, {Key: KeyConfidenceThreshold, Build: confidence_threshold.New}, {Key: KeyOriginalTextPresence, Build: original_text_presence.New}, {Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New}, diff --git a/internal/validators/registry_test.go b/internal/validators/registry_test.go index 6ae3880..0ef1ee8 100644 --- a/internal/validators/registry_test.go +++ b/internal/validators/registry_test.go @@ -14,6 +14,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/validators/no_effect" "gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text" "gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence" + "gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape" "gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms" "gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility" ) @@ -21,6 +22,7 @@ import ( func TestBuiltInRegistryRegistersAllKeys(t *testing.T) { r := NewBuiltInRegistry() for _, key := range []string{ + KeyProposalShape, KeyConfidenceThreshold, KeyOriginalTextPresence, KeyNonEmptyCorrectedText, @@ -52,6 +54,7 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) { wantClass validatormetadata.ExecutionClass } cases := []validatorCtor{ + {name: "proposal_shape", key: KeyProposalShape, build: proposal_shape.New, wantClass: validatormetadata.ExecutionClassDeterministic}, {name: "confidence_threshold", key: KeyConfidenceThreshold, build: confidence_threshold.New, wantClass: validatormetadata.ExecutionClassDeterministic}, {name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic}, {name: "non_empty_corrected_text", key: KeyNonEmptyCorrectedText, build: non_empty_corrected_text.New, wantClass: validatormetadata.ExecutionClassDeterministic},