1 Commits

Author SHA1 Message Date
a3655f5540 Make module-stage LLM handling resilient and report warnings 2026-05-23 10:07:06 -05:00
43 changed files with 856 additions and 217 deletions

View File

@@ -19,6 +19,7 @@ Pipeline behavior includes:
- conservative spoken-word dysfluency cleanup with semantic guardrails - conservative spoken-word dysfluency cleanup with semantic guardrails
- grammar/punctuation/capitalization/formatting cleanup - grammar/punctuation/capitalization/formatting cleanup
- validator-chain enforcement before application - validator-chain enforcement before application
- malformed module-stage LLM payloads degrade to warnings/rejections instead of failing the run
- run reports and diagnostics artifacts with secret redaction - run reports and diagnostics artifacts with secret redaction
## Build and Install ## Build and Install
@@ -130,6 +131,7 @@ audita process transcript.json \
- Without `--output`, stdout contains transcript JSON only on success. - Without `--output`, stdout contains transcript JSON only on success.
- `--report-json` writes a file and is never printed to stdout. - `--report-json` writes a file and is never printed to stdout.
- stderr is human-readable diagnostics/errors. - stderr is human-readable diagnostics/errors.
- successful runs remain quiet on stderr even when module warnings are recorded in report/diagnostics artifacts.
For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md). For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
@@ -149,10 +151,10 @@ audita config print-effective --config audita.yml
``` ```
For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md). For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md).
For output-schema details, see [`docs/output-schemas.md`](docs/output-schemas.md). For output-schema details, see [`docs/architecture/output-schemas.md`](docs/architecture/output-schemas.md).
For built-in validator keys and chain definitions, see [`docs/validators.md`](docs/validators.md). For built-in validator keys and chain definitions, see [`docs/architecture/validators.md`](docs/architecture/validators.md).
For embedded prompt assets and prompt metadata behavior, see [`docs/prompts.md`](docs/prompts.md). For embedded prompt assets and prompt metadata behavior, see [`docs/architecture/prompts.md`](docs/architecture/prompts.md).
For CLI/process compatibility guarantees, see [`docs/public-contract.md`](docs/public-contract.md). For CLI/process compatibility guarantees, see [`docs/architecture/public-contract.md`](docs/architecture/public-contract.md).
### Modules ### Modules

View File

@@ -382,25 +382,22 @@ func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testin
"--work-dir-retention", "--work-dir-retention",
"always", "always",
) )
if result.exitCode == 0 { if result.exitCode != 0 {
t.Fatalf("expected nonzero exit code") t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
} }
if result.stdout != "" { if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout) t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
} }
if !strings.Contains(result.stderr, "runner_execution") { if result.stderr != "" {
t.Fatalf("expected runner_execution failure, got %q", result.stderr) t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
if !strings.Contains(result.stderr, "diagnostics:") {
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
} }
report := readFile(t, reportPath) report := readFile(t, reportPath)
if !json.Valid(report) { if !json.Valid(report) {
t.Fatalf("expected valid failure report JSON") t.Fatalf("expected valid success report JSON")
} }
runDir := onlyRunDir(t, workDir) runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("expected error.log, got: %v", err) t.Fatalf("did not expect error.log, got: %v", err)
} }
} }

View File

@@ -133,11 +133,16 @@ internal/framework/validators/
llm_batching.go llm_batching.go
llm_validators.go llm_validators.go
internal/framework/warnings/
warnings.go
internal/validators/ internal/validators/
metadata/ metadata/
metadata.go metadata.go
registry.go registry.go
chains.go chains.go
proposal_shape/
validator.go
confidence_threshold/ confidence_threshold/
validator.go validator.go
original_text_presence/ original_text_presence/
@@ -242,6 +247,7 @@ Important behavior details:
- Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` continue to run production module paths with LLM-backed proposal generation and validator-chain execution. - Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` continue to run production module paths with LLM-backed proposal generation and validator-chain execution.
- Default runs (without explicit module selection) perform LLM calls through production module and validator paths. - Default runs (without explicit module selection) perform LLM calls through production module and validator paths.
- Success path is generally quiet on stderr. - Success path is generally quiet on stderr.
- Malformed module-stage LLM payloads degrade to validator rejections and module warnings instead of aborting the run.
- Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`. - Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`.
## Implemented data contracts ## Implemented data contracts
@@ -539,20 +545,20 @@ Validator execution classification metadata:
Stable built-in validator keys: Stable built-in validator keys:
- deterministic: - deterministic:
- `proposal_shape`
- `confidence_threshold` - `confidence_threshold`
- `original_text_presence` - `original_text_presence`
- `non_empty_corrected_text` - `non_empty_corrected_text` (historical key name; current semantics reject empty resulting segment text)
- `no_effect` - `no_effect`
- `protected_terms` - `protected_terms`
- LLM-backed: - LLM-backed:
- `spoken_form_plausibility` - `spoken_form_plausibility`
- `meaning_reversal_review` - `meaning_reversal_review`
- `editorial_review` - `editorial_review`
- `grammar_review`
- `spoken_word_review`
Built-in module chains: Built-in module chains:
- `glossary`: - `glossary`:
- `proposal_shape`
- `no_effect` - `no_effect`
- `original_text_presence` - `original_text_presence`
- `confidence_threshold` - `confidence_threshold`
@@ -561,6 +567,7 @@ Built-in module chains:
- `spoken_form_plausibility` - `spoken_form_plausibility`
- `meaning_reversal_review` - `meaning_reversal_review`
- `homophones`: - `homophones`:
- `proposal_shape`
- `no_effect` - `no_effect`
- `original_text_presence` - `original_text_presence`
- `confidence_threshold` - `confidence_threshold`
@@ -569,20 +576,22 @@ Built-in module chains:
- `spoken_form_plausibility` - `spoken_form_plausibility`
- `meaning_reversal_review` - `meaning_reversal_review`
- `spoken_word`: - `spoken_word`:
- `proposal_shape`
- `no_effect` - `no_effect`
- `original_text_presence` - `original_text_presence`
- `confidence_threshold` - `confidence_threshold`
- `protected_terms` - `protected_terms`
- `non_empty_corrected_text` - `non_empty_corrected_text`
- `spoken_word_review` - `editorial_review`
- `meaning_reversal_review` - `meaning_reversal_review`
- `grammar`: - `grammar`:
- `proposal_shape`
- `no_effect` - `no_effect`
- `original_text_presence` - `original_text_presence`
- `confidence_threshold` - `confidence_threshold`
- `protected_terms` - `protected_terms`
- `non_empty_corrected_text` - `non_empty_corrected_text`
- `grammar_review` - `editorial_review`
- `meaning_reversal_review` - `meaning_reversal_review`
1.0 boundary: 1.0 boundary:
@@ -595,12 +604,14 @@ Built-in module chains:
- prompt builders for: - prompt builders for:
- spoken-form plausibility - spoken-form plausibility
- meaning reversal detection - meaning reversal detection
- editorial review - editorial review
- grammar review - grammar review
- spoken-word review - spoken-word review
- deterministic batching by `validation_max_prompt_tokens`; - deterministic batching by `validation_max_prompt_tokens`;
- strict cardinality validation of structured LLM decisions (missing/duplicate/unknown indexes fail); - strict cardinality validation of synthesized validator decision sets;
- safe failure behavior for malformed/invalid structured responses. - malformed validator payloads reject only the affected batch with warnings;
- oversized single-proposal validator inputs reject only the affected proposal;
- transport/provider/runtime LLM call failures remain fatal.
`internal/framework/runner` wires LLM validators into existing validator chains using: `internal/framework/runner` wires LLM validators into existing validator chains using:
- the internal structured LLM client abstraction (`contracts.StructuredLLMClient`); - the internal structured LLM client abstraction (`contracts.StructuredLLMClient`);
@@ -726,6 +737,7 @@ Current process reports include diagnostics metadata references for:
Current process reports also include: Current process reports also include:
- module-level results (when runner modules execute), including applied/skipped proposal changes; - module-level results (when runner modules execute), including applied/skipped proposal changes;
- run-level module summary totals and failed module instance metadata. - run-level module summary totals and failed module instance metadata.
- module-level warning records for malformed proposal-generation payloads and malformed validator batches.
- module-level validator decisions and validator rejections. - module-level validator decisions and validator rejections.
- optional decision-level diagnostic artifact paths for validator LLM interactions when available. - optional decision-level diagnostic artifact paths for validator LLM interactions when available.
- stable validator keys in `validator_name` fields for validator decisions/rejections. - stable validator keys in `validator_name` fields for validator decisions/rejections.

View File

@@ -63,6 +63,7 @@ Ledger records are flattened review entries derived from module results and incl
- deterministic and LLM validator decision snapshots using stable validator keys. - deterministic and LLM validator decision snapshots using stable validator keys.
Validator rejection and proposal-application skip are distinct dispositions. Validator rejection and proposal-application skip are distinct dispositions.
Module warnings are reported in module results and diagnostics metadata, but do not create standalone correction-ledger rows.
## Report references ## Report references
@@ -71,6 +72,8 @@ Validator rejection and proposal-application skip are distinct dispositions.
- correction ledger artifact; - correction ledger artifact;
- existing transcript/normalization/chunking/invocation/effective-config artifacts. - existing transcript/normalization/chunking/invocation/effective-config artifacts.
Module report entries also include warning records for malformed proposal-generation payloads and malformed validator batches.
## Retention behavior ## Retention behavior
Run-directory retention follows configured policy: Run-directory retention follows configured policy:
@@ -93,6 +96,9 @@ When debugging:
- validator rejections: - validator rejections:
- inspect `correction-ledger.json` rejected entries and matching validator decisions; - inspect `correction-ledger.json` rejected entries and matching validator decisions;
- inspect validator response diagnostics payloads. - inspect validator response diagnostics payloads.
- module warnings:
- inspect module `warnings` entries in `report.json` or `--report-json`;
- follow any diagnostic artifact path on the warning to the recorded error/response payload.
- application skips: - application skips:
- inspect `correction-ledger.json` skipped entries and skip reason codes; - inspect `correction-ledger.json` skipped entries and skip reason codes;
- compare with validator decisions to distinguish validation rejection vs apply-time skip. - compare with validator decisions to distinguish validation rejection vs apply-time skip.

View File

@@ -93,6 +93,7 @@ Current values:
`--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata. `--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata.
Validator decision/rejection records in reports use stable validator keys in `validator_name`. Validator decision/rejection records in reports use stable validator keys in `validator_name`.
Module results may also include warning records for malformed module-stage LLM payloads.
Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds. Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds.
## Diagnostics directory behavior ## Diagnostics directory behavior
@@ -123,6 +124,7 @@ Success behavior:
- with `--output`, stdout is empty - with `--output`, stdout is empty
- without `--output`, stdout contains only transcript JSON in selected output schema - without `--output`, stdout contains only transcript JSON in selected output schema
- report JSON is not written to stdout - report JSON is not written to stdout
- success stderr remains empty even when reports/diagnostics contain module warnings
Failure behavior: Failure behavior:
- stderr contains human-readable error summary - stderr contains human-readable error summary
@@ -149,6 +151,7 @@ Config files should reference secrets via environment variable names (`api_key_e
## Compatibility and deprecation policy ## Compatibility and deprecation policy
- Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract. - Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract.
- Existing stable validator keys remain public contract values even when validator semantics are refined.
- Compatibility inputs (legacy flags/env aliases) may remain during transition windows. - Compatibility inputs (legacy flags/env aliases) may remain during transition windows.
- Any planned removal or behavior change should include clear compatibility notes and migration guidance. - Any planned removal or behavior change should include clear compatibility notes and migration guidance.

View File

@@ -8,6 +8,7 @@ For LLM-backed validator prompt asset details, see [`docs/prompts.md`](prompts.m
Built-in validator construction is package-owned under `internal/validators/<validator_key>`: Built-in validator construction is package-owned under `internal/validators/<validator_key>`:
- `internal/validators/confidence_threshold` - `internal/validators/confidence_threshold`
- `internal/validators/proposal_shape`
- `internal/validators/original_text_presence` - `internal/validators/original_text_presence`
- `internal/validators/non_empty_corrected_text` - `internal/validators/non_empty_corrected_text`
- `internal/validators/no_effect` - `internal/validators/no_effect`
@@ -45,12 +46,14 @@ Current 1.0 boundary:
### Deterministic validators ### Deterministic validators
- `proposal_shape`
- rejects malformed proposal fields before other validators run.
- `confidence_threshold` - `confidence_threshold`
- checks proposal confidence against module-specific configured threshold. - checks proposal confidence against module-specific configured threshold.
- `original_text_presence` - `original_text_presence`
- ensures target segment exists and `original_text` exists in current working segment text. - ensures target segment exists and `original_text` exists in current working segment text.
- `non_empty_corrected_text` - `non_empty_corrected_text`
- rejects blank/whitespace-only `corrected_text`. - rejects proposals whose previewed resulting segment text would be empty or whitespace-only.
- `no_effect` - `no_effect`
- rejects proposals where `original_text == corrected_text`. - rejects proposals where `original_text == corrected_text`.
- `protected_terms` - `protected_terms`
@@ -71,6 +74,7 @@ Current 1.0 boundary:
Current built-in chains resolved from `internal/validators/chains.go`: Current built-in chains resolved from `internal/validators/chains.go`:
- `glossary` - `glossary`
- `proposal_shape`
- `no_effect` - `no_effect`
- `original_text_presence` - `original_text_presence`
- `confidence_threshold` - `confidence_threshold`
@@ -80,6 +84,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
- `meaning_reversal_review` - `meaning_reversal_review`
- `homophones` - `homophones`
- `proposal_shape`
- `no_effect` - `no_effect`
- `original_text_presence` - `original_text_presence`
- `confidence_threshold` - `confidence_threshold`
@@ -89,6 +94,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
- `meaning_reversal_review` - `meaning_reversal_review`
- `spoken_word` - `spoken_word`
- `proposal_shape`
- `no_effect` - `no_effect`
- `original_text_presence` - `original_text_presence`
- `confidence_threshold` - `confidence_threshold`
@@ -98,6 +104,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
- `meaning_reversal_review` - `meaning_reversal_review`
- `grammar` - `grammar`
- `proposal_shape`
- `no_effect` - `no_effect`
- `original_text_presence` - `original_text_presence`
- `confidence_threshold` - `confidence_threshold`
@@ -119,7 +126,9 @@ Both variants preserve existing behavior and report the stable key `protected_te
- modules execute serially; - modules execute serially;
- section proposal work can run concurrently within a module; - section proposal work can run concurrently within a module;
- deterministic validators run before LLM-backed validators; - deterministic validators run before LLM-backed validators;
- malformed/missing/duplicate/unknown LLM validator decisions fail safely; - malformed module proposal payloads are downgraded to section-scoped module warnings with zero proposals for the affected section rather than module failure;
- malformed/missing/duplicate/unknown LLM validator decisions reject the affected validator batch with warnings instead of failing the module;
- oversized single-proposal validator inputs reject only the affected proposal under that validator;
- approved proposals are applied once per module after section work settles. - approved proposals are applied once per module after section work settles.
## Validator rejections vs proposal-application skips ## Validator rejections vs proposal-application skips
@@ -128,12 +137,15 @@ Both variants preserve existing behavior and report the stable key `protected_te
- proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code. - proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code.
- proposal-application skip: - proposal-application skip:
- proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time). - proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time).
- module warning:
- malformed proposal-generation payloads and malformed validator batches are recorded in module warning records and diagnostics without writing success stderr.
These are separate outcomes and are reported separately. These are separate outcomes and are reported separately.
## Reporting and diagnostics identity ## Reporting and diagnostics identity
- report validator decision/rejection entries use stable validator keys in `validator_name`. - report validator decision/rejection entries use stable validator keys in `validator_name`.
- report module results include warning records for malformed module-stage LLM payloads.
- validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata. - validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata.
- correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip. - correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip.

View File

@@ -40,6 +40,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
- Verify structured response schemas are attached via `response_format.type=json_schema`. - Verify structured response schemas are attached via `response_format.type=json_schema`.
- Verify diagnostics metadata includes structured schema `id/version/name/sha256`. - Verify diagnostics metadata includes structured schema `id/version/name/sha256`.
- Verify provider output is still locally decoded/validated before use. - Verify provider output is still locally decoded/validated before use.
- Verify malformed module-stage structured payloads degrade to warnings/rejections instead of failing the run.
## Report and diagnostics schema checks ## Report and diagnostics schema checks
@@ -67,6 +68,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
- Verify prompt metadata appears in LLM request metadata diagnostics: - Verify prompt metadata appears in LLM request metadata diagnostics:
- `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`. - `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`.
- Verify stable validator keys appear in report decisions/rejections. - Verify stable validator keys appear in report decisions/rejections.
- Verify module warning records appear in reports for malformed proposal-generation payloads and malformed validator batches.
- Verify built-in validator chains resolve and execute for default and explicit module runs. - Verify built-in validator chains resolve and execute for default and explicit module runs.
## Utilization diagnostics checks ## Utilization diagnostics checks
@@ -96,6 +98,8 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
## Failure and cancellation checks ## Failure and cancellation checks
- Verify controlled failure paths retain diagnostics and produce best-effort failure reports. - Verify controlled failure paths retain diagnostics and produce best-effort failure reports.
- Verify malformed proposal-generation payloads keep exit code `0`, keep stderr empty on success, and record warnings in reports/diagnostics.
- Verify malformed validator payloads reject only the affected batch and do not fail the module.
- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized. - Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized.
## Release fixture/idempotence checks ## Release fixture/idempotence checks

72
docs/roadmap/publish.md Normal file
View File

@@ -0,0 +1,72 @@
# Hard-Cutover Roadmap for Module-Stage LLM Resilience
## Summary
This roadmap captures the module-stage resilience work for Audita:
- fail fast on initialization, configuration, schema, and other pre-module setup errors;
- remain resilient during module execution when LLM payloads are malformed or individual proposed corrections are invalid;
- reject bad corrections through validator/reporting paths instead of aborting the module or process;
- keep success stderr quiet; surface warnings only through report and diagnostics artifacts;
- use a hard cutover only, with no compatibility aliases or transitional code.
Locked decisions:
- keep the stable public validator key `non_empty_corrected_text`;
- change that validators behavior to mean “the resulting segment text must not be empty/whitespace-only after applying the proposal preview”;
- malformed LLM-validator batch payloads reject the entire affected batch under that validator and continue;
- proposal/validator transport failures, timeouts, and provider/runtime call failures remain fatal;
- malformed structured payloads are downgraded; non-malformed runtime call failures are not.
## Stage 1: Proposal Intake Hardening
- Stop treating invalid individual structured corrections as fatal during proposal generation.
- Preserve returned correction ordering and proposal-index assignment even when individual corrections are malformed.
- Allow `corrected_text == ""` when the resulting segment remains non-empty after previewed application.
- Downgrade malformed proposal-generation structured payloads into section-scoped warnings with zero proposals for that section.
- Keep proposal-generation transport/provider/runtime call failures fatal.
Deterministic validation changes:
- Add `proposal_shape` as a built-in deterministic validator and run it first in every built-in module chain.
- Reject malformed proposal fields with stable reason codes:
- `invalid_target_segment_id`
- `empty_original_text`
- `invalid_confidence`
- Keep `non_empty_corrected_text` as the stable validator key, but change its semantics to reject only `empty_resulting_segment`.
- Keep validator rejection and apply-time skip as distinct outcomes.
## Stage 2: LLM Validator Resilience
- Keep validator transport/provider/runtime call failures fatal.
- Downgrade malformed validator structured payloads into batch-scoped validator rejections plus module warnings.
- Downgrade oversized single-proposal validator inputs into per-proposal validator rejections plus module warnings.
- Preserve decision-cardinality enforcement as an internal invariant after malformed-payload degradation has synthesized complete decision sets.
Stable reason codes introduced or relied upon by this cutover:
- `empty_resulting_segment`
- `invalid_target_segment_id`
- `empty_original_text`
- `invalid_confidence`
- `validator_response_malformed`
- `validator_input_too_large`
- `proposal_response_malformed`
## Stage 3: Reporting, Diagnostics, and Docs
- Add module warning records to runner results and process reports.
- Keep correction-ledger entries per-correction only; do not add standalone warning rows.
- Record malformed proposal-generation and validator-batch warnings through report and diagnostics artifacts only.
- Keep successful runs quiet on stderr even when warnings are present.
- Update README and architecture/public-contract/diagnostics/validators/release-checklist docs to reflect the new behavior.
## Acceptance Criteria
- Empty `corrected_text` may delete words, but proposals that would blank the whole segment are rejected or skipped safely.
- Invalid proposal shape is rejected by validators, not by proposal generation.
- Malformed proposal-generation payloads succeed with warnings and zero proposals for the affected section.
- Malformed validator payloads reject only the affected validator batch and do not fail the module.
- Oversized single validator inputs reject only the affected proposal.
- Transport/provider/runtime LLM failures still fail the module and process.
- Successful runs with warnings still exit `0`, emit transcript output normally, keep stderr empty, and expose warnings in report/diagnostics artifacts.

View File

@@ -26,6 +26,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
) )
type noOpStructuredLLMClient struct{} type noOpStructuredLLMClient struct{}
@@ -803,6 +804,7 @@ func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummar
ReplacementPolicy: string(r.ReplacementPolicy), ReplacementPolicy: string(r.ReplacementPolicy),
Status: r.Status, Status: r.Status,
ProposalCount: r.ProposalCount, ProposalCount: r.ProposalCount,
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions), ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected), ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
AppliedChanges: r.AppliedChanges, AppliedChanges: r.AppliedChanges,

View File

@@ -1793,11 +1793,12 @@ type fakeModule struct {
func (m fakeModule) Key() string { return m.key } func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return m.validators } func (m fakeModule) Validators() []contracts.Validator { return m.validators }
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if m.proposeF == nil { if m.proposeF == nil {
return nil, nil return contracts.ProposalResult{}, nil
} }
return m.proposeF(req) proposalsOut, err := m.proposeF(req)
return contracts.ProposalResult{Proposals: proposalsOut}, err
} }
type fakeValidator struct { type fakeValidator struct {
@@ -2384,7 +2385,7 @@ func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testi
} }
} }
func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing.T) { func TestRunProcessExplicitGrammarMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")} processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil }) t.Cleanup(func() { processProposalLLMClient = nil })
@@ -2403,22 +2404,32 @@ func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing
"--work-dir-retention", "always", "--work-dir-retention", "always",
"--report-json", reportPath, "--report-json", reportPath,
}, &stdout, &stderr) }, &stdout, &stderr)
if exitCode == 0 { if exitCode != 0 {
t.Fatal("expected failure") t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
} }
if stdout.Len() != 0 { if stderr.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) t.Fatalf("expected empty stderr on success, got %q", stderr.String())
} }
if !strings.Contains(stderr.String(), "runner_execution") { parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
t.Fatalf("expected runner_execution error, got %q", stderr.String()) if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
} }
runDir := onlyRunDir(t, workDir) runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("expected error.log on failed grammar run: %v", err) t.Fatalf("did not expect error.log on successful grammar run: %v", err)
} }
report := readProcessReport(t, reportPath) report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" { if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected failed runner_execution report, got %+v", report) t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
}
if report.ModuleResults[0].Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warning: %+v", report.ModuleResults[0].Warnings[0])
} }
} }
@@ -3041,7 +3052,7 @@ func TestRunProcessExplicitGlossaryRepeatedStagesUseDeterministicInstanceNamesAn
} }
} }
func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testing.T) { func TestRunProcessExplicitGlossaryMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")} processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil }) t.Cleanup(func() { processProposalLLMClient = nil })
@@ -3060,22 +3071,29 @@ func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testin
"--work-dir-retention", "always", "--work-dir-retention", "always",
"--report-json", reportPath, "--report-json", reportPath,
}, &stdout, &stderr) }, &stdout, &stderr)
if exitCode == 0 { if exitCode != 0 {
t.Fatal("expected failure") t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
} }
if stdout.Len() != 0 { if stderr.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) t.Fatalf("expected empty stderr on success, got %q", stderr.String())
} }
if !strings.Contains(stderr.String(), "runner_execution") { parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
t.Fatalf("expected runner_execution error, got %q", stderr.String()) if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
} }
runDir := onlyRunDir(t, workDir) runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("expected error.log on failed glossary run: %v", err) t.Fatalf("did not expect error.log on successful glossary run: %v", err)
} }
report := readProcessReport(t, reportPath) report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" { if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected failed runner_execution report, got %+v", report) t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
} }
} }
@@ -3295,7 +3313,7 @@ func TestRunProcessExplicitHomophonesProtectedGlossaryTermRejected(t *testing.T)
} }
} }
func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *testing.T) { func TestRunProcessExplicitHomophonesMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")} processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil }) t.Cleanup(func() { processProposalLLMClient = nil })
@@ -3314,22 +3332,29 @@ func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *test
"--work-dir-retention", "always", "--work-dir-retention", "always",
"--report-json", reportPath, "--report-json", reportPath,
}, &stdout, &stderr) }, &stdout, &stderr)
if exitCode == 0 { if exitCode != 0 {
t.Fatal("expected failure") t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
} }
if stdout.Len() != 0 { if stderr.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) t.Fatalf("expected empty stderr on success, got %q", stderr.String())
} }
if !strings.Contains(stderr.String(), "runner_execution") { parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
t.Fatalf("expected runner_execution error, got %q", stderr.String()) if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
} }
runDir := onlyRunDir(t, workDir) runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("expected error.log on failed homophones run: %v", err) t.Fatalf("did not expect error.log on successful homophones run: %v", err)
} }
report := readProcessReport(t, reportPath) report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" { if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected failed runner_execution report, got %+v", report) t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
} }
} }
@@ -3674,7 +3699,7 @@ func TestRunProcessExplicitSpokenWordProtectedGlossaryTermRejected(t *testing.T)
} }
} }
func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *testing.T) { func TestRunProcessExplicitSpokenWordMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")} processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil }) t.Cleanup(func() { processProposalLLMClient = nil })
@@ -3693,22 +3718,29 @@ func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *test
"--work-dir-retention", "always", "--work-dir-retention", "always",
"--report-json", reportPath, "--report-json", reportPath,
}, &stdout, &stderr) }, &stdout, &stderr)
if exitCode == 0 { if exitCode != 0 {
t.Fatal("expected failure") t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
} }
if stdout.Len() != 0 { if stderr.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) t.Fatalf("expected empty stderr on success, got %q", stderr.String())
} }
if !strings.Contains(stderr.String(), "runner_execution") { parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
t.Fatalf("expected runner_execution error, got %q", stderr.String()) if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
} }
runDir := onlyRunDir(t, workDir) runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil { if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("expected error.log on failed spoken_word run: %v", err) t.Fatalf("did not expect error.log on successful spoken_word run: %v", err)
} }
report := readProcessReport(t, reportPath) report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" { if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected failed runner_execution report, got %+v", report) t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
} }
} }

View File

@@ -78,25 +78,22 @@ func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req co
} }
} }
case "mid_pipeline_fail": case "mid_pipeline_fail":
switch target := out.(type) { if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
case *proposal_generation.StructuredCorrectionSet:
c.mu.Lock() c.mu.Lock()
c.proposals++ c.proposals++
proposalCall := c.proposals proposalCall := c.proposals
c.mu.Unlock() c.mu.Unlock()
if proposalCall >= 3 { if proposalCall >= 3 {
*target = proposal_generation.StructuredCorrectionSet{ return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
Corrections: []proposal_generation.StructuredCorrectionProposal{ }
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99}, }
}, switch target := out.(type) {
} case *proposal_generation.StructuredCorrectionSet:
} else { *target = proposal_generation.StructuredCorrectionSet{
*target = proposal_generation.StructuredCorrectionSet{ Corrections: []proposal_generation.StructuredCorrectionProposal{
Corrections: []proposal_generation.StructuredCorrectionProposal{ {TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99}, },
},
}
} }
case *validators.LLMValidationResponse: case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: nil} *target = validators.LLMValidationResponse{Validations: nil}

View File

@@ -7,6 +7,7 @@ import (
"time" "time"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
) )
type ProcessReport struct { type ProcessReport struct {
@@ -46,18 +47,19 @@ type ReportMetadata struct {
} }
type ModuleReport struct { type ModuleReport struct {
ModuleKey string `json:"module_key"` ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"` ModuleInstance string `json:"module_instance"`
ReplacementPolicy string `json:"replacement_policy,omitempty"` ReplacementPolicy string `json:"replacement_policy,omitempty"`
Status string `json:"status"` Status string `json:"status"`
ProposalCount int `json:"proposal_count"` ProposalCount int `json:"proposal_count"`
ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"` Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"` ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"`
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"` AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
ErrorMessage string `json:"error_message,omitempty"` SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"` ErrorMessage string `json:"error_message,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"` StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
} }
type ValidatorDecisionReport struct { type ValidatorDecisionReport struct {

View File

@@ -47,7 +47,7 @@ func (h testChunkProposalHarness) collectEnrichedProposals(
return nil, err return nil, err
} }
for _, proposal := range base { for _, proposal := range base.Proposals {
sectionIndex := section.Index sectionIndex := section.Index
out = append(out, proposals.EnrichedCorrectionProposal{ out = append(out, proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposal, CorrectionProposal: proposal,
@@ -85,11 +85,11 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy
func (m deterministicFakeModule) Validators() []Validator { return nil } func (m deterministicFakeModule) Validators() []Validator { return nil }
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) { func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
_ = ctx _ = ctx
if req.WorkingTranscript == nil || req.Section == nil { if req.WorkingTranscript == nil || req.Section == nil {
return []proposals.CorrectionProposal{}, nil return ProposalResult{}, nil
} }
out := make([]proposals.CorrectionProposal, 0) out := make([]proposals.CorrectionProposal, 0)
@@ -110,7 +110,7 @@ func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalReques
} }
} }
return out, nil return ProposalResult{Proposals: out}, nil
} }
func TestChunkProposalMetadataAssociation(t *testing.T) { func TestChunkProposalMetadataAssociation(t *testing.T) {

View File

@@ -11,6 +11,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
) )
// StructuredLLMClient provides provider-agnostic structured completion. // StructuredLLMClient provides provider-agnostic structured completion.
@@ -28,7 +29,7 @@ type TranscriptModule interface {
Key() string Key() string
ReplacementPolicy() proposals.ReplacementPolicy ReplacementPolicy() proposals.ReplacementPolicy
Validators() []Validator Validators() []Validator
Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error)
} }
// Validator evaluates candidate proposals and returns one decision per proposal index. // Validator evaluates candidate proposals and returns one decision per proposal index.
@@ -103,6 +104,11 @@ type ProposalRequest struct {
LLMScheduler LLMScheduler `json:"-"` LLMScheduler LLMScheduler `json:"-"`
} }
type ProposalResult struct {
Proposals []proposals.CorrectionProposal `json:"proposals,omitempty"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
}
// ValidationRequest is the input to validator execution. // ValidationRequest is the input to validator execution.
type ValidationRequest = validators.Request type ValidationRequest = validators.Request

View File

@@ -50,11 +50,13 @@ func (f *fakeModule) Validators() []Validator {
return []Validator{&fakeValidator{}} return []Validator{&fakeValidator{}}
} }
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) { func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
_ = ctx _ = ctx
_ = req _ = req
return []proposals.CorrectionProposal{ return ProposalResult{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9}, Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
},
}, nil }, nil
} }
@@ -68,12 +70,12 @@ func TestInterfaceContractsCompileWithFakes(t *testing.T) {
t.Fatalf("unexpected module key: %q", got) t.Fatalf("unexpected module key: %q", got)
} }
proposalsOut, err := module.Propose(context.Background(), ProposalRequest{}) proposalResult, err := module.Propose(context.Background(), ProposalRequest{})
if err != nil { if err != nil {
t.Fatalf("unexpected propose error: %v", err) t.Fatalf("unexpected propose error: %v", err)
} }
if len(proposalsOut) != 1 { if len(proposalResult.Proposals) != 1 {
t.Fatalf("expected one proposal, got %d", len(proposalsOut)) t.Fatalf("expected one proposal, got %d", len(proposalResult.Proposals))
} }
} }

View File

@@ -19,10 +19,10 @@ func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique return proposals.ReplacementPolicyRequireUnique
} }
func (m noopModule) Validators() []contracts.Validator { return nil } func (m noopModule) Validators() []contracts.Validator { return nil }
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
_ = ctx _ = ctx
_ = req _ = req
return nil, nil return contracts.ProposalResult{}, nil
} }
func TestKnownModuleKeyRecognition(t *testing.T) { func TestKnownModuleKeyRecognition(t *testing.T) {

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
) )
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts. // InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
@@ -68,6 +69,7 @@ type Request struct {
type Result struct { type Result struct {
Corrections []proposals.CorrectionProposal `json:"corrections"` Corrections []proposals.CorrectionProposal `json:"corrections"`
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"` Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
Artifacts InteractionArtifacts `json:"artifacts,omitempty"` Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
} }
@@ -156,6 +158,12 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
} }
if callErr != nil { if callErr != nil {
if isMalformedStructuredOutputError(callErr) {
return Result{
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
Artifacts: artifacts,
}, nil
}
return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr) return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr)
} }
@@ -168,9 +176,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
CorrectedText: raw.CorrectedText, CorrectedText: raw.CorrectedText,
Confidence: raw.Confidence, Confidence: raw.Confidence,
} }
if err := candidate.Validate(); err != nil {
return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err)
}
corrections = append(corrections, candidate) corrections = append(corrections, candidate)
enrichedCandidate := proposals.EnrichedCorrectionProposal{ enrichedCandidate := proposals.EnrichedCorrectionProposal{
@@ -191,6 +196,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
return Result{ return Result{
Corrections: corrections, Corrections: corrections,
Enriched: enriched, Enriched: enriched,
Warnings: nil,
Artifacts: artifacts, Artifacts: artifacts,
}, nil }, nil
} }
@@ -240,6 +246,48 @@ func errPayload(err error) any {
return map[string]any{"error": err.Error()} return map[string]any{"error": err.Error()}
} }
func isMalformedStructuredOutputError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
warning := stagewarnings.StageWarning{
Scope: stagewarnings.ScopeProposalGeneration,
ReasonCode: "proposal_response_malformed",
Message: strings.TrimSpace(err.Error()),
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
}
if section != nil {
sectionIndex := section.Index
warning.SectionIndex = &sectionIndex
}
return warning
}
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
if artifacts.ErrorPayloadPath != "" {
return artifacts.ErrorPayloadPath
}
return artifacts.ResponsePayloadPath
}
type diagnosticsWriterAdapter struct { type diagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter writer *llm.DiagnosticsWriter
} }

View File

@@ -224,12 +224,12 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
} }
} }
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) { func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) {
client := &fakeStructuredClient{ client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{ responses: []StructuredCorrectionSet{
{ {
Corrections: []StructuredCorrectionProposal{ Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9}, {TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
}, },
}, },
}, },
@@ -237,9 +237,38 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
req := defaultRequest(t) req := defaultRequest(t)
req.LLMClient = client req.LLMClient = client
_, err := GenerateCandidates(context.Background(), req) result, err := GenerateCandidates(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "invalid structured correction") { if err != nil {
t.Fatalf("expected structured response validation failure, got %v", err) t.Fatalf("expected invalid correction to survive generation, got %v", err)
}
if len(result.Corrections) != 1 {
t.Fatalf("expected one correction, got %+v", result)
}
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
}
if len(result.Warnings) != 0 {
t.Fatalf("did not expect warnings for individually invalid corrections, got %+v", result.Warnings)
}
}
func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) {
client := &fakeStructuredClient{err: errors.New("malformed structured output")}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed structured output to downgrade to warning, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Warnings) != 1 {
t.Fatalf("expected one warning, got %+v", result.Warnings)
}
if result.Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warning: %+v", result.Warnings[0])
} }
} }

View File

@@ -147,6 +147,8 @@ func skipReasonMessage(reason ProposalSkipReason) string {
return "original_text matched multiple spans under require_unique policy" return "original_text matched multiple spans under require_unique policy"
case SkipReasonNoEffect: case SkipReasonNoEffect:
return "original_text and corrected_text are identical" return "original_text and corrected_text are identical"
case SkipReasonEmptyResultingText:
return "proposal would leave the segment empty"
case SkipReasonInvalidProposal: case SkipReasonInvalidProposal:
return "proposal failed structural or policy validation" return "proposal failed structural or policy validation"
default: default:

View File

@@ -15,6 +15,7 @@ const (
SkipReasonMissingOriginalText ProposalSkipReason = "missing_original_text" SkipReasonMissingOriginalText ProposalSkipReason = "missing_original_text"
SkipReasonAmbiguousOriginal ProposalSkipReason = "ambiguous_original_text" SkipReasonAmbiguousOriginal ProposalSkipReason = "ambiguous_original_text"
SkipReasonNoEffect ProposalSkipReason = "no_effect" SkipReasonNoEffect ProposalSkipReason = "no_effect"
SkipReasonEmptyResultingText ProposalSkipReason = "empty_resulting_segment"
SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal" SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal"
) )
@@ -62,6 +63,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
} }
corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1) corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
}
return SegmentPreviewResult{ return SegmentPreviewResult{
Applicable: true, Applicable: true,
CorrectedSegmentText: corrected, CorrectedSegmentText: corrected,
@@ -70,6 +74,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
case ReplacementPolicyReplaceAll: case ReplacementPolicyReplaceAll:
corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText) corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
}
return SegmentPreviewResult{ return SegmentPreviewResult{
Applicable: true, Applicable: true,
CorrectedSegmentText: corrected, CorrectedSegmentText: corrected,

View File

@@ -121,6 +121,42 @@ func TestPreviewProposalForSegmentNoEffectReplacement(t *testing.T) {
} }
} }
func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonEmpty(t *testing.T) {
segment := &schema.Segment{ID: 8, Text: "uh hello"}
proposal := CorrectionProposal{
TargetSegmentID: 8,
OriginalText: "uh ",
CorrectedText: "",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
if !result.Applicable {
t.Fatalf("expected applicable preview, got skip reason %q", result.SkipReason)
}
if result.CorrectedSegmentText != "hello" {
t.Fatalf("unexpected corrected text: %q", result.CorrectedSegmentText)
}
}
func TestPreviewProposalForSegmentRejectsEmptyResultingSegment(t *testing.T) {
segment := &schema.Segment{ID: 8, Text: "uh"}
proposal := CorrectionProposal{
TargetSegmentID: 8,
OriginalText: "uh",
CorrectedText: "",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
if result.Applicable {
t.Fatal("expected non-applicable preview")
}
if result.SkipReason != SkipReasonEmptyResultingText {
t.Fatalf("expected skip reason %q, got %q", SkipReasonEmptyResultingText, result.SkipReason)
}
}
func TestPreviewProposalForSegmentPreservesInputSegment(t *testing.T) { func TestPreviewProposalForSegmentPreservesInputSegment(t *testing.T) {
segment := &schema.Segment{ID: 9, Speaker: "A", Start: 1.0, End: 2.0, Text: "rank rank"} segment := &schema.Segment{ID: 9, Speaker: "A", Start: 1.0, End: 2.0, Text: "rank rank"}
original := *segment original := *segment

View File

@@ -38,9 +38,6 @@ func (p CorrectionProposal) Validate() error {
if strings.TrimSpace(p.OriginalText) == "" { if strings.TrimSpace(p.OriginalText) == "" {
return fmt.Errorf("proposal original_text must not be empty") return fmt.Errorf("proposal original_text must not be empty")
} }
if strings.TrimSpace(p.CorrectedText) == "" {
return fmt.Errorf("proposal corrected_text must not be empty")
}
if p.Confidence < 0.0 || p.Confidence > 1.0 { if p.Confidence < 0.0 || p.Confidence > 1.0 {
return fmt.Errorf("proposal confidence must be between 0.0 and 1.0") return fmt.Errorf("proposal confidence must be between 0.0 and 1.0")
} }

View File

@@ -35,7 +35,7 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) {
} }
} }
func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) { func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
proposal := CorrectionProposal{ proposal := CorrectionProposal{
TargetSegmentID: 42, TargetSegmentID: 42,
OriginalText: "gestures", OriginalText: "gestures",
@@ -43,12 +43,8 @@ func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
Confidence: 0.95, Confidence: 0.95,
} }
err := proposal.Validate() if err := proposal.Validate(); err != nil {
if err == nil { t.Fatalf("expected empty corrected_text to be allowed, got %v", err)
t.Fatal("expected validation error, got nil")
}
if err.Error() != "proposal corrected_text must not be empty" {
t.Fatalf("unexpected error: %v", err)
} }
} }

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata" validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
) )
@@ -37,18 +38,19 @@ type ValidationScheduler = contracts.LLMScheduler
// ModuleResult captures deterministic per-module execution output. // ModuleResult captures deterministic per-module execution output.
type ModuleResult struct { type ModuleResult struct {
ModuleKey string `json:"module_key"` ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"` ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Status string `json:"status"` Status string `json:"status"`
ProposalCount int `json:"proposal_count"` ProposalCount int `json:"proposal_count"`
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"` Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"` ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"` AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
ErrorMessage string `json:"error_message,omitempty"` SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
StartedAt time.Time `json:"started_at"` ErrorMessage string `json:"error_message,omitempty"`
CompletedAt time.Time `json:"completed_at"` StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
} }
type ValidatorDecisionRecord struct { type ValidatorDecisionRecord struct {
@@ -176,6 +178,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
ReplacementPolicy: policy, ReplacementPolicy: policy,
Status: ModuleStatusFailed, Status: ModuleStatusFailed,
ProposalCount: pipelineResult.ProposalCount, ProposalCount: pipelineResult.ProposalCount,
Warnings: pipelineResult.Warnings,
ValidatorDecisions: pipelineResult.ValidatorDecisions, ValidatorDecisions: pipelineResult.ValidatorDecisions,
ValidatorRejected: pipelineResult.ValidatorRejected, ValidatorRejected: pipelineResult.ValidatorRejected,
ErrorMessage: pipelineErr.Error(), ErrorMessage: pipelineErr.Error(),
@@ -199,6 +202,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
ReplacementPolicy: policy, ReplacementPolicy: policy,
Status: ModuleStatusSuccess, Status: ModuleStatusSuccess,
ProposalCount: pipelineResult.ProposalCount, ProposalCount: pipelineResult.ProposalCount,
Warnings: pipelineResult.Warnings,
ValidatorDecisions: pipelineResult.ValidatorDecisions, ValidatorDecisions: pipelineResult.ValidatorDecisions,
ValidatorRejected: pipelineResult.ValidatorRejected, ValidatorRejected: pipelineResult.ValidatorRejected,
AppliedChanges: applyResult.Applied, AppliedChanges: applyResult.Applied,
@@ -232,12 +236,14 @@ type collectSectionProposalsInput struct {
type sectionProposals struct { type sectionProposals struct {
meta contracts.SectionMetadata meta contracts.SectionMetadata
corrected []proposals.CorrectionProposal corrected []proposals.CorrectionProposal
warnings []stagewarnings.StageWarning
} }
type sectionProposalResult struct { type sectionProposalResult struct {
sectionPos int sectionPos int
section chunking.Section section chunking.Section
corrected []proposals.CorrectionProposal corrected []proposals.CorrectionProposal
warnings []stagewarnings.StageWarning
err error err error
} }
@@ -247,12 +253,14 @@ type sectionValidationResult struct {
approved []proposals.EnrichedCorrectionProposal approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange rejected []ValidatorRejectedChange
warnings []stagewarnings.StageWarning
err error err error
} }
type modulePipelineResult struct { type modulePipelineResult struct {
ProposalCount int ProposalCount int
Approved []proposals.EnrichedCorrectionProposal Approved []proposals.EnrichedCorrectionProposal
Warnings []stagewarnings.StageWarning
ValidatorDecisions []ValidatorDecisionRecord ValidatorDecisions []ValidatorDecisionRecord
ValidatorRejected []ValidatorRejectedChange ValidatorRejected []ValidatorRejectedChange
} }
@@ -271,7 +279,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
go func() { go func() {
defer wg.Done() defer wg.Done()
meta := contracts.SectionMetadataFromSection(section) meta := contracts.SectionMetadataFromSection(section)
corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{ proposalResult, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{ ExecutionContext: contracts.ExecutionContext{
Config: input.Config, Config: input.Config,
WorkingTranscript: transcriptFromSection(section), WorkingTranscript: transcriptFromSection(section),
@@ -291,7 +299,8 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
case results <- sectionProposalResult{ case results <- sectionProposalResult{
sectionPos: sectionPos, sectionPos: sectionPos,
section: section, section: section,
corrected: corrected, corrected: proposalResult.Proposals,
warnings: proposalResult.Warnings,
err: err, err: err,
}: }:
case <-runCtx.Done(): case <-runCtx.Done():
@@ -308,6 +317,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) { func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
out := modulePipelineResult{ out := modulePipelineResult{
Approved: make([]proposals.EnrichedCorrectionProposal, 0), Approved: make([]proposals.EnrichedCorrectionProposal, 0),
Warnings: make([]stagewarnings.StageWarning, 0),
ValidatorDecisions: make([]ValidatorDecisionRecord, 0), ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
ValidatorRejected: make([]ValidatorRejectedChange, 0), ValidatorRejected: make([]ValidatorRejectedChange, 0),
} }
@@ -352,6 +362,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if firstErr != nil { if firstErr != nil {
continue continue
} }
out.Warnings = append(out.Warnings, result.warnings...)
pending[result.sectionPos] = result pending[result.sectionPos] = result
for { for {
@@ -401,6 +412,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
approved: validated.approved, approved: validated.approved,
decisions: validated.decisions, decisions: validated.decisions,
rejected: validated.rejected, rejected: validated.rejected,
warnings: validated.warnings,
err: err, err: err,
} }
}(nextSectionToProcess, sectionEnriched, sectionMeta) }(nextSectionToProcess, sectionEnriched, sectionMeta)
@@ -424,6 +436,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
break break
} }
out.Approved = append(out.Approved, res.approved...) out.Approved = append(out.Approved, res.approved...)
out.Warnings = append(out.Warnings, res.warnings...)
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...) out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...) out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
} }
@@ -440,6 +453,38 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
} }
return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName] return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName]
}) })
sort.SliceStable(out.Warnings, func(i, j int) bool {
leftSection, rightSection := -1, -1
if out.Warnings[i].SectionIndex != nil {
leftSection = *out.Warnings[i].SectionIndex
}
if out.Warnings[j].SectionIndex != nil {
rightSection = *out.Warnings[j].SectionIndex
}
if leftSection != rightSection {
return leftSection < rightSection
}
leftBatch, rightBatch := -1, -1
if out.Warnings[i].BatchIndex != nil {
leftBatch = *out.Warnings[i].BatchIndex
}
if out.Warnings[j].BatchIndex != nil {
rightBatch = *out.Warnings[j].BatchIndex
}
if leftBatch != rightBatch {
return leftBatch < rightBatch
}
if out.Warnings[i].ValidatorName != out.Warnings[j].ValidatorName {
return validatorOrder[out.Warnings[i].ValidatorName] < validatorOrder[out.Warnings[j].ValidatorName]
}
if out.Warnings[i].Scope != out.Warnings[j].Scope {
return out.Warnings[i].Scope < out.Warnings[j].Scope
}
if out.Warnings[i].ReasonCode != out.Warnings[j].ReasonCode {
return out.Warnings[i].ReasonCode < out.Warnings[j].ReasonCode
}
return out.Warnings[i].Message < out.Warnings[j].Message
})
if firstErr != nil { if firstErr != nil {
return out, firstErr return out, firstErr
@@ -467,11 +512,13 @@ type validateSectionCandidatesResult struct {
approved []proposals.EnrichedCorrectionProposal approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange rejected []ValidatorRejectedChange
warnings []stagewarnings.StageWarning
} }
func validateSectionCandidates(ctx context.Context, input validateSectionCandidatesInput) (validateSectionCandidatesResult, error) { func validateSectionCandidates(ctx context.Context, input validateSectionCandidatesInput) (validateSectionCandidatesResult, error) {
decisions := make([]ValidatorDecisionRecord, 0) decisions := make([]ValidatorDecisionRecord, 0)
rejected := make([]ValidatorRejectedChange, 0) rejected := make([]ValidatorRejectedChange, 0)
warnings := make([]stagewarnings.StageWarning, 0)
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...) eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
for _, validator := range input.Validators { for _, validator := range input.Validators {
@@ -512,6 +559,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
approved: eligible, approved: eligible,
decisions: decisions, decisions: decisions,
rejected: rejected, rejected: rejected,
warnings: warnings,
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err) }, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
} }
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil { if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
@@ -519,8 +567,10 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
approved: eligible, approved: eligible,
decisions: decisions, decisions: decisions,
rejected: rejected, rejected: rejected,
warnings: warnings,
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err) }, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err)
} }
warnings = append(warnings, vResult.Warnings...)
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible)) nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible)) byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
@@ -562,6 +612,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
approved: eligible, approved: eligible,
decisions: decisions, decisions: decisions,
rejected: rejected, rejected: rejected,
warnings: warnings,
}, nil }, nil
} }

View File

@@ -47,11 +47,12 @@ type fakeModule struct {
func (m fakeModule) Key() string { return m.key } func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return m.validators } func (m fakeModule) Validators() []contracts.Validator { return m.validators }
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if m.proposeF == nil { if m.proposeF == nil {
return nil, nil return contracts.ProposalResult{}, nil
} }
return m.proposeF(req) proposalsOut, err := m.proposeF(req)
return contracts.ProposalResult{Proposals: proposalsOut}, err
} }
type fakeValidator struct { type fakeValidator struct {
@@ -1191,7 +1192,7 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
} }
} }
func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) { func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}} client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1209,15 +1210,21 @@ func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
ValidationLLMClient: client, ValidationLLMClient: client,
}) })
if err == nil { if err != nil {
t.Fatal("expected llm validator failure") t.Fatalf("expected malformed validator response to downgrade, got %v", err)
} }
if out.FinalTranscript.Segments[0].Text != "the cat" { if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected partial progress retained") t.Fatalf("expected partial progress retained")
} }
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
}
if len(out.ModuleResults[1].Warnings) != 1 || out.ModuleResults[1].Warnings[0].ReasonCode != validators.ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", out.ModuleResults[1].Warnings)
}
} }
func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) { func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}} client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "") llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1232,12 +1239,12 @@ func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client, ValidationLLMClient: client,
}) })
if err == nil { if err != nil {
t.Fatal("expected missing decision failure") t.Fatalf("expected missing decision downgrade, got %v", err)
} }
} }
func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) { func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
@@ -1255,8 +1262,8 @@ func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client, ValidationLLMClient: client,
}) })
if err == nil { if err != nil {
t.Fatal("expected duplicate decision failure") t.Fatalf("expected duplicate decision downgrade, got %v", err)
} }
} }
@@ -1355,7 +1362,7 @@ type proposalGenerationModule struct {
func (m proposalGenerationModule) Key() string { return m.key } func (m proposalGenerationModule) Key() string { return m.key }
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil } func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey, ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName, ModuleInstance: req.RunSpec.InstanceName,
@@ -1372,9 +1379,9 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
DiagnosticsDir: req.DiagnosticsDir, DiagnosticsDir: req.DiagnosticsDir,
}) })
if err != nil { if err != nil {
return nil, err return contracts.ProposalResult{}, err
} }
return result.Corrections, nil return contracts.ProposalResult{Proposals: result.Corrections, Warnings: result.Warnings}, nil
} }
type fakeProposalStructuredClient struct { type fakeProposalStructuredClient struct {

View File

@@ -4,8 +4,35 @@ import (
"context" "context"
"fmt" "fmt"
"strings" "strings"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
) )
type ProposalShapeValidator struct{}
func (v ProposalShapeValidator) Name() string { return "proposal_shape" }
func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result, error) {
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
switch {
case c.TargetSegmentID <= 0:
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
case strings.TrimSpace(c.OriginalText) == "":
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
case c.Confidence < 0.0 || c.Confidence > 1.0:
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
default:
decisions = append(decisions, approval(c.ProposalIndex))
}
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type ConfidenceThresholdValidator struct{} type ConfidenceThresholdValidator struct{}
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" } func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
@@ -62,10 +89,23 @@ type NonEmptyCorrectionValidator struct{}
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" } func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" }
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) { func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
segmentsByID := make(map[int]schema.Segment)
if req.WorkingTranscript != nil {
for _, seg := range req.WorkingTranscript.Segments {
segmentsByID[seg.ID] = seg
}
}
decisions := make([]Decision, 0, len(req.CandidateProposal)) decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal { for _, c := range req.CandidateProposal {
if strings.TrimSpace(c.CorrectedText) == "" { segment, ok := segmentsByID[c.TargetSegmentID]
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty")) if !ok {
decisions = append(decisions, approval(c.ProposalIndex))
continue
}
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy)
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
continue continue
} }
decisions = append(decisions, approval(c.ProposalIndex)) decisions = append(decisions, approval(c.ProposalIndex))

View File

@@ -11,6 +11,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
"gitea.maximumdirect.net/eric/audita/internal/prompts" "gitea.maximumdirect.net/eric/audita/internal/prompts"
) )
@@ -78,7 +79,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
maxTokens = req.Config.ValidationMaxPromptTokens maxTokens = req.Config.ValidationMaxPromptTokens
} }
batches, err := ChunkLLMValidationItems(validationReq.Items, maxTokens, v.estimator) warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
if len(itemsForBatching) == 0 {
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
}
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
if err != nil { if err != nil {
return Result{}, err return Result{}, err
} }
@@ -140,12 +154,19 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
) )
} }
if err != nil { if err != nil {
if isMalformedStructuredOutputError(err) {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
continue
}
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
} }
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response) batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
if err != nil { if err != nil {
return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err) llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
continue
} }
for i := range batchDecisions { for i := range batchDecisions {
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
@@ -154,12 +175,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
} }
all := append([]Decision(nil), immediate...) all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
all = append(all, llmDecisions...) all = append(all, llmDecisions...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err return Result{}, err
} }
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all}, nil return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
} }
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata { func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
@@ -301,3 +323,98 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
} }
return decisions, nil return decisions, nil
} }
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
out := make([]Decision, 0)
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
}
return out
}
func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
out := make([]stagewarnings.StageWarning, 0)
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
ReasonCode: ReasonValidatorInputTooLarge,
Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
})
}
return out
}
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
if len(decisions) == 0 {
return append([]LLMValidationItem(nil), items...)
}
rejected := make(map[int]struct{}, len(decisions))
for _, decision := range decisions {
rejected[decision.ProposalIndex] = struct{}{}
}
out := make([]LLMValidationItem, 0, len(items))
for _, item := range items {
if _, ok := rejected[item.CorrectionIndex]; ok {
continue
}
out = append(out, item)
}
return out
}
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
out := make([]Decision, 0, len(items))
for _, item := range items {
out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
}
return out
}
func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
idx := batchIndex
return stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
BatchIndex: &idx,
ReasonCode: reasonCode,
Message: strings.TrimSpace(message),
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
}
}
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
if artifacts.ErrorPayloadPath != "" {
return artifacts.ErrorPayloadPath
}
return artifacts.ResponsePayloadPath
}
func isMalformedStructuredOutputError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}

View File

@@ -248,29 +248,38 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
} }
} }
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) { func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")} client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client req.LLMClient = client
_, err := v.Validate(context.Background(), req) res, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "completion failed") { if err != nil {
t.Fatalf("expected malformed output error, got %v", err) t.Fatalf("expected malformed output downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
} }
} }
func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) { func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}} client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client req.LLMClient = client
_, err := v.Validate(context.Background(), req) res, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "response invalid") { if err != nil {
t.Fatalf("expected missing decision error, got %v", err) t.Fatalf("expected missing decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
} }
} }
func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) { func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
@@ -278,20 +287,74 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client req.LLMClient = client
_, err := v.Validate(context.Background(), req) res, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "duplicate") { if err != nil {
t.Fatalf("expected duplicate decision error, got %v", err) t.Fatalf("expected duplicate decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
} }
} }
func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) { func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}} client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client req.LLMClient = client
_, err := v.Validate(context.Background(), req) res, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "unknown") { if err != nil {
t.Fatalf("expected unknown index error, got %v", err) t.Fatalf("expected unknown index downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
huge := strings.Repeat("gestures ", 200)
req := Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Text: huge},
{ID: 2, Text: "There were gestures at the temple.", Categories: []string{"narration"}},
}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
},
ModuleKey: "homophones",
ModuleInstance: "homophones",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
}
req.LLMClient = client
cfg := config.Default()
cfg.ValidationMaxPromptTokens = 200
req.Config = &cfg
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected oversize downgrade, got %v", err)
}
if len(res.Decisions) != 2 {
t.Fatalf("expected two decisions, got %+v", res.Decisions)
}
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved {
t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
}
if !res.Decisions[1].Approved {
t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("expected one oversize warning, got %+v", res.Warnings)
} }
} }

View File

@@ -8,16 +8,22 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
) )
const ( const (
ReasonApproved = "approved" ReasonApproved = "approved"
ReasonLowConfidence = "low_confidence" ReasonLowConfidence = "low_confidence"
ReasonMissingOriginalText = "missing_original_text" ReasonMissingOriginalText = "missing_original_text"
ReasonMissingTargetSegment = "missing_target_segment" ReasonMissingTargetSegment = "missing_target_segment"
ReasonEmptyCorrectedText = "empty_corrected_text" ReasonEmptyResultingText = "empty_resulting_segment"
ReasonNoEffect = "no_effect" ReasonNoEffect = "no_effect"
ReasonProtectedGlossaryTerm = "protected_glossary_term" ReasonProtectedGlossaryTerm = "protected_glossary_term"
ReasonInvalidTargetSegment = "invalid_target_segment_id"
ReasonEmptyOriginalText = "empty_original_text"
ReasonInvalidConfidence = "invalid_confidence"
ReasonValidatorMalformed = "validator_response_malformed"
ReasonValidatorInputTooLarge = "validator_input_too_large"
) )
// Request is the runtime input shared by deterministic validators. // Request is the runtime input shared by deterministic validators.
@@ -45,8 +51,9 @@ type Decision struct {
// Result is one validator output containing exactly one decision per proposal index. // Result is one validator output containing exactly one decision per proposal index.
type Result struct { type Result struct {
ValidatorName string `json:"validator_name"` ValidatorName string `json:"validator_name"`
Decisions []Decision `json:"decisions"` Decisions []Decision `json:"decisions"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
} }
// ValidationScheduler provides bounded execution for validator LLM calls. // ValidationScheduler provides bounded execution for validator LLM calls.

View File

@@ -68,6 +68,31 @@ func TestConfidenceThresholdValidator(t *testing.T) {
} }
} }
func TestProposalShapeValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 0, "teh", "the", 0.9),
mkCandidate(2, 1, " ", "the", 0.9),
mkCandidate(3, 1, "teh", "the", 1.5),
}}
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved {
t.Fatalf("expected proposal 0 approved")
}
if res.Decisions[1].ReasonCode != ReasonInvalidTargetSegment {
t.Fatalf("expected invalid target segment rejection, got %+v", res.Decisions[1])
}
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
}
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
}
}
func TestOriginalTextPresenceValidator(t *testing.T) { func TestOriginalTextPresenceValidator(t *testing.T) {
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{ req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9), mkCandidate(0, 1, "hello", "hi", 0.9),
@@ -90,10 +115,13 @@ func TestOriginalTextPresenceValidator(t *testing.T) {
} }
func TestNonEmptyCorrectionValidator(t *testing.T) { func TestNonEmptyCorrectionValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{ req := Request{
mkCandidate(0, 1, "hello", "hi", 0.9), WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
mkCandidate(1, 1, "hello", " ", 0.9), ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
}} CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "hello world", " ", 0.9),
}}
res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req) res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req)
if err != nil { if err != nil {
t.Fatalf("Validate error: %v", err) t.Fatalf("Validate error: %v", err)
@@ -101,8 +129,8 @@ func TestNonEmptyCorrectionValidator(t *testing.T) {
if !res.Decisions[0].Approved { if !res.Decisions[0].Approved {
t.Fatalf("expected proposal 0 approved") t.Fatalf("expected proposal 0 approved")
} }
if res.Decisions[1].ReasonCode != ReasonEmptyCorrectedText { if res.Decisions[1].ReasonCode != ReasonEmptyResultingText {
t.Fatalf("expected empty_corrected_text, got %+v", res.Decisions[1]) t.Fatalf("expected empty_resulting_segment, got %+v", res.Decisions[1])
} }
} }
@@ -151,9 +179,14 @@ func TestStableReasonCodes(t *testing.T) {
ReasonLowConfidence, ReasonLowConfidence,
ReasonMissingOriginalText, ReasonMissingOriginalText,
ReasonMissingTargetSegment, ReasonMissingTargetSegment,
ReasonEmptyCorrectedText, ReasonEmptyResultingText,
ReasonNoEffect, ReasonNoEffect,
ReasonProtectedGlossaryTerm, ReasonProtectedGlossaryTerm,
ReasonInvalidTargetSegment,
ReasonEmptyOriginalText,
ReasonInvalidConfidence,
ReasonValidatorMalformed,
ReasonValidatorInputTooLarge,
} }
for _, code := range codes { for _, code := range codes {
if strings.TrimSpace(code) == "" { if strings.TrimSpace(code) == "" {

View File

@@ -0,0 +1,18 @@
package warnings
type Scope string
const (
ScopeProposalGeneration Scope = "proposal_generation"
ScopeValidator Scope = "validator"
)
type StageWarning struct {
Scope Scope `json:"scope"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
SectionIndex *int `json:"section_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
BatchIndex *int `json:"batch_index,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}

View File

@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...) return append([]contracts.Validator(nil), m.validators...)
} }
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section) sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0 sectionIndex := 0
if req.Section != nil { if req.Section != nil {
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
} }
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription) messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil { if err != nil {
return nil, err return contracts.ProposalResult{}, err
} }
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
DiagnosticsDir: req.DiagnosticsDir, DiagnosticsDir: req.DiagnosticsDir,
}) })
if err != nil { if err != nil {
return nil, err return contracts.ProposalResult{}, err
} }
return generated.Corrections, nil return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
} }
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript { func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {

View File

@@ -129,6 +129,7 @@ func TestGlossaryModuleValidatorChain(t *testing.T) {
got = append(got, v.Name()) got = append(got, v.Name())
} }
want := []string{ want := []string{
"proposal_shape",
"no_effect", "no_effect",
"original_text_presence", "original_text_presence",
"confidence_threshold", "confidence_threshold",
@@ -185,7 +186,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
if len(client.calls) != 1 || client.calls[0].StageName != "glossary:proposal" { if len(client.calls) != 1 || client.calls[0].StageName != "glossary:proposal" {
t.Fatalf("expected one glossary:proposal call, got %+v", client.calls) t.Fatalf("expected one glossary:proposal call, got %+v", client.calls)
} }
if len(out) != 1 || out[0].CorrectedText != "Jesters" { if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
t.Fatalf("unexpected proposals: %+v", out) t.Fatalf("unexpected proposals: %+v", out)
} }
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "glossary", "*proposal*response-payload.json")) diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "glossary", "*proposal*response-payload.json"))

View File

@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...) return append([]contracts.Validator(nil), m.validators...)
} }
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section) sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0 sectionIndex := 0
if req.Section != nil { if req.Section != nil {
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
} }
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription) messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil { if err != nil {
return nil, err return contracts.ProposalResult{}, err
} }
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
DiagnosticsDir: req.DiagnosticsDir, DiagnosticsDir: req.DiagnosticsDir,
}) })
if err != nil { if err != nil {
return nil, err return contracts.ProposalResult{}, err
} }
return generated.Corrections, nil return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
} }
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript { func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {

View File

@@ -131,6 +131,7 @@ func TestGrammarModuleValidatorChain(t *testing.T) {
got = append(got, v.Name()) got = append(got, v.Name())
} }
want := []string{ want := []string{
"proposal_shape",
"no_effect", "no_effect",
"original_text_presence", "original_text_presence",
"confidence_threshold", "confidence_threshold",
@@ -187,7 +188,7 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal" { if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal" {
t.Fatalf("expected one grammar:proposal call, got %+v", client.calls) t.Fatalf("expected one grammar:proposal call, got %+v", client.calls)
} }
if len(out) != 1 || out[0].CorrectedText != "Hello, world" { if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Hello, world" {
t.Fatalf("unexpected proposals: %+v", out) t.Fatalf("unexpected proposals: %+v", out)
} }
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "grammar", "*proposal*response-payload.json")) diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "grammar", "*proposal*response-payload.json"))

View File

@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...) return append([]contracts.Validator(nil), m.validators...)
} }
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section) sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0 sectionIndex := 0
if req.Section != nil { if req.Section != nil {
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
} }
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription) messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil { if err != nil {
return nil, err return contracts.ProposalResult{}, err
} }
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
DiagnosticsDir: req.DiagnosticsDir, DiagnosticsDir: req.DiagnosticsDir,
}) })
if err != nil { if err != nil {
return nil, err return contracts.ProposalResult{}, err
} }
return generated.Corrections, nil return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
} }
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript { func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {

View File

@@ -145,6 +145,7 @@ func TestHomophonesModuleValidatorChain(t *testing.T) {
got = append(got, v.Name()) got = append(got, v.Name())
} }
want := []string{ want := []string{
"proposal_shape",
"no_effect", "no_effect",
"original_text_presence", "original_text_presence",
"confidence_threshold", "confidence_threshold",
@@ -201,7 +202,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" { if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" {
t.Fatalf("expected one homophones:proposal call, got %+v", client.calls) t.Fatalf("expected one homophones:proposal call, got %+v", client.calls)
} }
if len(out) != 1 || out[0].CorrectedText != "Jesters" { if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
t.Fatalf("unexpected proposals: %+v", out) t.Fatalf("unexpected proposals: %+v", out)
} }
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json")) diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json"))

View File

@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...) return append([]contracts.Validator(nil), m.validators...)
} }
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section) sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0 sectionIndex := 0
if req.Section != nil { if req.Section != nil {
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
} }
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription) messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil { if err != nil {
return nil, err return contracts.ProposalResult{}, err
} }
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
DiagnosticsDir: req.DiagnosticsDir, DiagnosticsDir: req.DiagnosticsDir,
}) })
if err != nil { if err != nil {
return nil, err return contracts.ProposalResult{}, err
} }
return generated.Corrections, nil return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
} }
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript { func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {

View File

@@ -132,6 +132,7 @@ func TestSpokenWordModuleValidatorChain(t *testing.T) {
got = append(got, v.Name()) got = append(got, v.Name())
} }
want := []string{ want := []string{
"proposal_shape",
"no_effect", "no_effect",
"original_text_presence", "original_text_presence",
"confidence_threshold", "confidence_threshold",
@@ -188,7 +189,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
if len(client.calls) != 1 || client.calls[0].StageName != "spoken_word:proposal" { if len(client.calls) != 1 || client.calls[0].StageName != "spoken_word:proposal" {
t.Fatalf("expected one spoken_word:proposal call, got %+v", client.calls) t.Fatalf("expected one spoken_word:proposal call, got %+v", client.calls)
} }
if len(out) != 1 || out[0].CorrectedText != "I think" { if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "I think" {
t.Fatalf("unexpected proposals: %+v", out) t.Fatalf("unexpected proposals: %+v", out)
} }
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "spoken_word", "*proposal*response-payload.json")) diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "spoken_word", "*proposal*response-payload.json"))

View File

@@ -9,6 +9,7 @@ import (
var builtInChains = map[string][]string{ var builtInChains = map[string][]string{
"glossary": { "glossary": {
KeyProposalShape,
KeyNoEffect, KeyNoEffect,
KeyOriginalTextPresence, KeyOriginalTextPresence,
KeyConfidenceThreshold, KeyConfidenceThreshold,
@@ -18,6 +19,7 @@ var builtInChains = map[string][]string{
KeyMeaningReversalReview, KeyMeaningReversalReview,
}, },
"homophones": { "homophones": {
KeyProposalShape,
KeyNoEffect, KeyNoEffect,
KeyOriginalTextPresence, KeyOriginalTextPresence,
KeyConfidenceThreshold, KeyConfidenceThreshold,
@@ -27,6 +29,7 @@ var builtInChains = map[string][]string{
KeyMeaningReversalReview, KeyMeaningReversalReview,
}, },
"spoken_word": { "spoken_word": {
KeyProposalShape,
KeyNoEffect, KeyNoEffect,
KeyOriginalTextPresence, KeyOriginalTextPresence,
KeyConfidenceThreshold, KeyConfidenceThreshold,
@@ -36,6 +39,7 @@ var builtInChains = map[string][]string{
KeyMeaningReversalReview, KeyMeaningReversalReview,
}, },
"grammar": { "grammar": {
KeyProposalShape,
KeyNoEffect, KeyNoEffect,
KeyOriginalTextPresence, KeyOriginalTextPresence,
KeyConfidenceThreshold, KeyConfidenceThreshold,

View File

@@ -0,0 +1,13 @@
package proposal_shape
import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
const Key = "proposal_shape"
func New() (contracts.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.ProposalShapeValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -11,11 +11,13 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect" "gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text" "gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence" "gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
"gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape"
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms" "gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
"gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility" "gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility"
) )
const ( const (
KeyProposalShape = "proposal_shape"
KeyConfidenceThreshold = "confidence_threshold" KeyConfidenceThreshold = "confidence_threshold"
KeyOriginalTextPresence = "original_text_presence" KeyOriginalTextPresence = "original_text_presence"
KeyNonEmptyCorrectedText = "non_empty_corrected_text" KeyNonEmptyCorrectedText = "non_empty_corrected_text"
@@ -39,6 +41,7 @@ type Registry struct {
func NewBuiltInRegistry() *Registry { func NewBuiltInRegistry() *Registry {
defs := []BuiltInValidatorDefinition{ defs := []BuiltInValidatorDefinition{
{Key: KeyProposalShape, Build: proposal_shape.New},
{Key: KeyConfidenceThreshold, Build: confidence_threshold.New}, {Key: KeyConfidenceThreshold, Build: confidence_threshold.New},
{Key: KeyOriginalTextPresence, Build: original_text_presence.New}, {Key: KeyOriginalTextPresence, Build: original_text_presence.New},
{Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New}, {Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New},

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect" "gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text" "gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence" "gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
"gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape"
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms" "gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
"gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility" "gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility"
) )
@@ -21,6 +22,7 @@ import (
func TestBuiltInRegistryRegistersAllKeys(t *testing.T) { func TestBuiltInRegistryRegistersAllKeys(t *testing.T) {
r := NewBuiltInRegistry() r := NewBuiltInRegistry()
for _, key := range []string{ for _, key := range []string{
KeyProposalShape,
KeyConfidenceThreshold, KeyConfidenceThreshold,
KeyOriginalTextPresence, KeyOriginalTextPresence,
KeyNonEmptyCorrectedText, KeyNonEmptyCorrectedText,
@@ -52,6 +54,7 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
wantClass validatormetadata.ExecutionClass wantClass validatormetadata.ExecutionClass
} }
cases := []validatorCtor{ cases := []validatorCtor{
{name: "proposal_shape", key: KeyProposalShape, build: proposal_shape.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "confidence_threshold", key: KeyConfidenceThreshold, build: confidence_threshold.New, wantClass: validatormetadata.ExecutionClassDeterministic}, {name: "confidence_threshold", key: KeyConfidenceThreshold, build: confidence_threshold.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic}, {name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "non_empty_corrected_text", key: KeyNonEmptyCorrectedText, build: non_empty_corrected_text.New, wantClass: validatormetadata.ExecutionClassDeterministic}, {name: "non_empty_corrected_text", key: KeyNonEmptyCorrectedText, build: non_empty_corrected_text.New, wantClass: validatormetadata.ExecutionClassDeterministic},