Make module-stage LLM handling resilient and report warnings

This commit is contained in:
2026-05-23 10:07:06 -05:00
parent a84941d681
commit a3655f5540
43 changed files with 856 additions and 217 deletions

View File

@@ -19,6 +19,7 @@ Pipeline behavior includes:
- conservative spoken-word dysfluency cleanup with semantic guardrails
- grammar/punctuation/capitalization/formatting cleanup
- validator-chain enforcement before application
- malformed module-stage LLM payloads degrade to warnings/rejections instead of failing the run
- run reports and diagnostics artifacts with secret redaction
## Build and Install
@@ -130,6 +131,7 @@ audita process transcript.json \
- Without `--output`, stdout contains transcript JSON only on success.
- `--report-json` writes a file and is never printed to stdout.
- stderr is human-readable diagnostics/errors.
- successful runs remain quiet on stderr even when module warnings are recorded in report/diagnostics artifacts.
For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
@@ -149,10 +151,10 @@ audita config print-effective --config audita.yml
```
For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md).
For output-schema details, see [`docs/output-schemas.md`](docs/output-schemas.md).
For built-in validator keys and chain definitions, see [`docs/validators.md`](docs/validators.md).
For embedded prompt assets and prompt metadata behavior, see [`docs/prompts.md`](docs/prompts.md).
For CLI/process compatibility guarantees, see [`docs/public-contract.md`](docs/public-contract.md).
For output-schema details, see [`docs/architecture/output-schemas.md`](docs/architecture/output-schemas.md).
For built-in validator keys and chain definitions, see [`docs/architecture/validators.md`](docs/architecture/validators.md).
For embedded prompt assets and prompt metadata behavior, see [`docs/architecture/prompts.md`](docs/architecture/prompts.md).
For CLI/process compatibility guarantees, see [`docs/architecture/public-contract.md`](docs/architecture/public-contract.md).
### Modules

View File

@@ -382,25 +382,22 @@ func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testin
"--work-dir-retention",
"always",
)
if result.exitCode == 0 {
t.Fatalf("expected nonzero exit code")
if result.exitCode != 0 {
t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
}
if result.stdout != "" {
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
if !json.Valid([]byte(result.stdout)) {
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
}
if !strings.Contains(result.stderr, "runner_execution") {
t.Fatalf("expected runner_execution failure, got %q", result.stderr)
}
if !strings.Contains(result.stderr, "diagnostics:") {
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
if result.stderr != "" {
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
}
report := readFile(t, reportPath)
if !json.Valid(report) {
t.Fatalf("expected valid failure report JSON")
t.Fatalf("expected valid success report JSON")
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log, got: %v", err)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log, got: %v", err)
}
}

View File

@@ -133,11 +133,16 @@ internal/framework/validators/
llm_batching.go
llm_validators.go
internal/framework/warnings/
warnings.go
internal/validators/
metadata/
metadata.go
registry.go
chains.go
proposal_shape/
validator.go
confidence_threshold/
validator.go
original_text_presence/
@@ -242,6 +247,7 @@ Important behavior details:
- Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` continue to run production module paths with LLM-backed proposal generation and validator-chain execution.
- Default runs (without explicit module selection) perform LLM calls through production module and validator paths.
- Success path is generally quiet on stderr.
- Malformed module-stage LLM payloads degrade to validator rejections and module warnings instead of aborting the run.
- Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`.
## Implemented data contracts
@@ -539,20 +545,20 @@ Validator execution classification metadata:
Stable built-in validator keys:
- deterministic:
- `proposal_shape`
- `confidence_threshold`
- `original_text_presence`
- `non_empty_corrected_text`
- `non_empty_corrected_text` (historical key name; current semantics reject empty resulting segment text)
- `no_effect`
- `protected_terms`
- LLM-backed:
- `spoken_form_plausibility`
- `meaning_reversal_review`
- `editorial_review`
- `grammar_review`
- `spoken_word_review`
Built-in module chains:
- `glossary`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
@@ -561,6 +567,7 @@ Built-in module chains:
- `spoken_form_plausibility`
- `meaning_reversal_review`
- `homophones`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
@@ -569,20 +576,22 @@ Built-in module chains:
- `spoken_form_plausibility`
- `meaning_reversal_review`
- `spoken_word`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `spoken_word_review`
- `editorial_review`
- `meaning_reversal_review`
- `grammar`:
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
- `protected_terms`
- `non_empty_corrected_text`
- `grammar_review`
- `editorial_review`
- `meaning_reversal_review`
1.0 boundary:
@@ -595,12 +604,14 @@ Built-in module chains:
- prompt builders for:
- spoken-form plausibility
- meaning reversal detection
- editorial review
- grammar review
- spoken-word review
- editorial review
- grammar review
- spoken-word review
- deterministic batching by `validation_max_prompt_tokens`;
- strict cardinality validation of structured LLM decisions (missing/duplicate/unknown indexes fail);
- safe failure behavior for malformed/invalid structured responses.
- strict cardinality validation of synthesized validator decision sets;
- malformed validator payloads reject only the affected batch with warnings;
- oversized single-proposal validator inputs reject only the affected proposal;
- transport/provider/runtime LLM call failures remain fatal.
`internal/framework/runner` wires LLM validators into existing validator chains using:
- the internal structured LLM client abstraction (`contracts.StructuredLLMClient`);
@@ -726,6 +737,7 @@ Current process reports include diagnostics metadata references for:
Current process reports also include:
- module-level results (when runner modules execute), including applied/skipped proposal changes;
- run-level module summary totals and failed module instance metadata.
- module-level warning records for malformed proposal-generation payloads and malformed validator batches.
- module-level validator decisions and validator rejections.
- optional decision-level diagnostic artifact paths for validator LLM interactions when available.
- stable validator keys in `validator_name` fields for validator decisions/rejections.

View File

@@ -63,6 +63,7 @@ Ledger records are flattened review entries derived from module results and incl
- deterministic and LLM validator decision snapshots using stable validator keys.
Validator rejection and proposal-application skip are distinct dispositions.
Module warnings are reported in module results and diagnostics metadata, but do not create standalone correction-ledger rows.
## Report references
@@ -71,6 +72,8 @@ Validator rejection and proposal-application skip are distinct dispositions.
- correction ledger artifact;
- existing transcript/normalization/chunking/invocation/effective-config artifacts.
Module report entries also include warning records for malformed proposal-generation payloads and malformed validator batches.
## Retention behavior
Run-directory retention follows configured policy:
@@ -93,6 +96,9 @@ When debugging:
- validator rejections:
- inspect `correction-ledger.json` rejected entries and matching validator decisions;
- inspect validator response diagnostics payloads.
- module warnings:
- inspect module `warnings` entries in `report.json` or `--report-json`;
- follow any diagnostic artifact path on the warning to the recorded error/response payload.
- application skips:
- inspect `correction-ledger.json` skipped entries and skip reason codes;
- compare with validator decisions to distinguish validation rejection vs apply-time skip.

View File

@@ -93,6 +93,7 @@ Current values:
`--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata.
Validator decision/rejection records in reports use stable validator keys in `validator_name`.
Module results may also include warning records for malformed module-stage LLM payloads.
Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds.
## Diagnostics directory behavior
@@ -123,6 +124,7 @@ Success behavior:
- with `--output`, stdout is empty
- without `--output`, stdout contains only transcript JSON in selected output schema
- report JSON is not written to stdout
- success stderr remains empty even when reports/diagnostics contain module warnings
Failure behavior:
- stderr contains human-readable error summary
@@ -149,6 +151,7 @@ Config files should reference secrets via environment variable names (`api_key_e
## Compatibility and deprecation policy
- Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract.
- Existing stable validator keys remain public contract values even when validator semantics are refined.
- Compatibility inputs (legacy flags/env aliases) may remain during transition windows.
- Any planned removal or behavior change should include clear compatibility notes and migration guidance.

View File

@@ -8,6 +8,7 @@ For LLM-backed validator prompt asset details, see [`docs/prompts.md`](prompts.m
Built-in validator construction is package-owned under `internal/validators/<validator_key>`:
- `internal/validators/confidence_threshold`
- `internal/validators/proposal_shape`
- `internal/validators/original_text_presence`
- `internal/validators/non_empty_corrected_text`
- `internal/validators/no_effect`
@@ -45,12 +46,14 @@ Current 1.0 boundary:
### Deterministic validators
- `proposal_shape`
- rejects malformed proposal fields before other validators run.
- `confidence_threshold`
- checks proposal confidence against module-specific configured threshold.
- `original_text_presence`
- ensures target segment exists and `original_text` exists in current working segment text.
- `non_empty_corrected_text`
- rejects blank/whitespace-only `corrected_text`.
- rejects proposals whose previewed resulting segment text would be empty or whitespace-only.
- `no_effect`
- rejects proposals where `original_text == corrected_text`.
- `protected_terms`
@@ -71,6 +74,7 @@ Current 1.0 boundary:
Current built-in chains resolved from `internal/validators/chains.go`:
- `glossary`
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
@@ -80,6 +84,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
- `meaning_reversal_review`
- `homophones`
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
@@ -89,6 +94,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
- `meaning_reversal_review`
- `spoken_word`
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
@@ -98,6 +104,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
- `meaning_reversal_review`
- `grammar`
- `proposal_shape`
- `no_effect`
- `original_text_presence`
- `confidence_threshold`
@@ -119,7 +126,9 @@ Both variants preserve existing behavior and report the stable key `protected_te
- modules execute serially;
- section proposal work can run concurrently within a module;
- deterministic validators run before LLM-backed validators;
- malformed/missing/duplicate/unknown LLM validator decisions fail safely;
- malformed module proposal payloads are downgraded to section-scoped module warnings with zero proposals for the affected section rather than module failure;
- malformed/missing/duplicate/unknown LLM validator decisions reject the affected validator batch with warnings instead of failing the module;
- oversized single-proposal validator inputs reject only the affected proposal under that validator;
- approved proposals are applied once per module after section work settles.
## Validator rejections vs proposal-application skips
@@ -128,12 +137,15 @@ Both variants preserve existing behavior and report the stable key `protected_te
- proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code.
- proposal-application skip:
- proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time).
- module warning:
- malformed proposal-generation payloads and malformed validator batches are recorded in module warning records and diagnostics without writing success stderr.
These are separate outcomes and are reported separately.
## Reporting and diagnostics identity
- report validator decision/rejection entries use stable validator keys in `validator_name`.
- report module results include warning records for malformed module-stage LLM payloads.
- validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata.
- correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip.

View File

@@ -40,6 +40,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
- Verify structured response schemas are attached via `response_format.type=json_schema`.
- Verify diagnostics metadata includes structured schema `id/version/name/sha256`.
- Verify provider output is still locally decoded/validated before use.
- Verify malformed module-stage structured payloads degrade to warnings/rejections instead of failing the run.
## Report and diagnostics schema checks
@@ -67,6 +68,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
- Verify prompt metadata appears in LLM request metadata diagnostics:
- `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`.
- Verify stable validator keys appear in report decisions/rejections.
- Verify module warning records appear in reports for malformed proposal-generation payloads and malformed validator batches.
- Verify built-in validator chains resolve and execute for default and explicit module runs.
## Utilization diagnostics checks
@@ -96,6 +98,8 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
## Failure and cancellation checks
- Verify controlled failure paths retain diagnostics and produce best-effort failure reports.
- Verify malformed proposal-generation payloads keep exit code `0`, keep stderr empty on success, and record warnings in reports/diagnostics.
- Verify malformed validator payloads reject only the affected batch and do not fail the module.
- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized.
## Release fixture/idempotence checks

72
docs/roadmap/publish.md Normal file
View File

@@ -0,0 +1,72 @@
# Hard-Cutover Roadmap for Module-Stage LLM Resilience
## Summary
This roadmap captures the module-stage resilience work for Audita:
- fail fast on initialization, configuration, schema, and other pre-module setup errors;
- remain resilient during module execution when LLM payloads are malformed or individual proposed corrections are invalid;
- reject bad corrections through validator/reporting paths instead of aborting the module or process;
- keep success stderr quiet; surface warnings only through report and diagnostics artifacts;
- use a hard cutover only, with no compatibility aliases or transitional code.
Locked decisions:
- keep the stable public validator key `non_empty_corrected_text`;
- change that validators behavior to mean “the resulting segment text must not be empty/whitespace-only after applying the proposal preview”;
- malformed LLM-validator batch payloads reject the entire affected batch under that validator and continue;
- proposal/validator transport failures, timeouts, and provider/runtime call failures remain fatal;
- malformed structured payloads are downgraded; non-malformed runtime call failures are not.
## Stage 1: Proposal Intake Hardening
- Stop treating invalid individual structured corrections as fatal during proposal generation.
- Preserve returned correction ordering and proposal-index assignment even when individual corrections are malformed.
- Allow `corrected_text == ""` when the resulting segment remains non-empty after previewed application.
- Downgrade malformed proposal-generation structured payloads into section-scoped warnings with zero proposals for that section.
- Keep proposal-generation transport/provider/runtime call failures fatal.
Deterministic validation changes:
- Add `proposal_shape` as a built-in deterministic validator and run it first in every built-in module chain.
- Reject malformed proposal fields with stable reason codes:
- `invalid_target_segment_id`
- `empty_original_text`
- `invalid_confidence`
- Keep `non_empty_corrected_text` as the stable validator key, but change its semantics to reject only `empty_resulting_segment`.
- Keep validator rejection and apply-time skip as distinct outcomes.
## Stage 2: LLM Validator Resilience
- Keep validator transport/provider/runtime call failures fatal.
- Downgrade malformed validator structured payloads into batch-scoped validator rejections plus module warnings.
- Downgrade oversized single-proposal validator inputs into per-proposal validator rejections plus module warnings.
- Preserve decision-cardinality enforcement as an internal invariant after malformed-payload degradation has synthesized complete decision sets.
Stable reason codes introduced or relied upon by this cutover:
- `empty_resulting_segment`
- `invalid_target_segment_id`
- `empty_original_text`
- `invalid_confidence`
- `validator_response_malformed`
- `validator_input_too_large`
- `proposal_response_malformed`
## Stage 3: Reporting, Diagnostics, and Docs
- Add module warning records to runner results and process reports.
- Keep correction-ledger entries per-correction only; do not add standalone warning rows.
- Record malformed proposal-generation and validator-batch warnings through report and diagnostics artifacts only.
- Keep successful runs quiet on stderr even when warnings are present.
- Update README and architecture/public-contract/diagnostics/validators/release-checklist docs to reflect the new behavior.
## Acceptance Criteria
- Empty `corrected_text` may delete words, but proposals that would blank the whole segment are rejected or skipped safely.
- Invalid proposal shape is rejected by validators, not by proposal generation.
- Malformed proposal-generation payloads succeed with warnings and zero proposals for the affected section.
- Malformed validator payloads reject only the affected validator batch and do not fail the module.
- Oversized single validator inputs reject only the affected proposal.
- Transport/provider/runtime LLM failures still fail the module and process.
- Successful runs with warnings still exit `0`, emit transcript output normally, keep stderr empty, and expose warnings in report/diagnostics artifacts.

View File

@@ -26,6 +26,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
type noOpStructuredLLMClient struct{}
@@ -803,6 +804,7 @@ func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummar
ReplacementPolicy: string(r.ReplacementPolicy),
Status: r.Status,
ProposalCount: r.ProposalCount,
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
AppliedChanges: r.AppliedChanges,

View File

@@ -1793,11 +1793,12 @@ type fakeModule struct {
func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if m.proposeF == nil {
return nil, nil
return contracts.ProposalResult{}, nil
}
return m.proposeF(req)
proposalsOut, err := m.proposeF(req)
return contracts.ProposalResult{Proposals: proposalsOut}, err
}
type fakeValidator struct {
@@ -2384,7 +2385,7 @@ func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testi
}
}
func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
func TestRunProcessExplicitGrammarMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
@@ -2403,22 +2404,32 @@ func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
if stderr.Len() != 0 {
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed grammar run: %v", err)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log on successful grammar run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
}
if report.ModuleResults[0].Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warning: %+v", report.ModuleResults[0].Warnings[0])
}
}
@@ -3041,7 +3052,7 @@ func TestRunProcessExplicitGlossaryRepeatedStagesUseDeterministicInstanceNamesAn
}
}
func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
func TestRunProcessExplicitGlossaryMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
@@ -3060,22 +3071,29 @@ func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testin
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
if stderr.Len() != 0 {
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed glossary run: %v", err)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log on successful glossary run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
}
}
@@ -3295,7 +3313,7 @@ func TestRunProcessExplicitHomophonesProtectedGlossaryTermRejected(t *testing.T)
}
}
func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
func TestRunProcessExplicitHomophonesMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
@@ -3314,22 +3332,29 @@ func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *test
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
if stderr.Len() != 0 {
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed homophones run: %v", err)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log on successful homophones run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
}
}
@@ -3674,7 +3699,7 @@ func TestRunProcessExplicitSpokenWordProtectedGlossaryTermRejected(t *testing.T)
}
}
func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
func TestRunProcessExplicitSpokenWordMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
@@ -3693,22 +3718,29 @@ func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *test
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
if stderr.Len() != 0 {
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
if err != nil {
t.Fatalf("expected transcript stdout on success: %v", err)
}
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed spoken_word run: %v", err)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
t.Fatalf("did not expect error.log on successful spoken_word run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
if report.Status != "success" || report.ErrorPhase != "" {
t.Fatalf("expected successful report, got %+v", report)
}
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
}
}

View File

@@ -78,25 +78,22 @@ func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req co
}
}
case "mid_pipeline_fail":
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
c.mu.Lock()
c.proposals++
proposalCall := c.proposals
c.mu.Unlock()
if proposalCall >= 3 {
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
},
}
} else {
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
},
}
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
}
}
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
},
}
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: nil}

View File

@@ -7,6 +7,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
type ProcessReport struct {
@@ -46,18 +47,19 @@ type ReportMetadata struct {
}
type ModuleReport struct {
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy string `json:"replacement_policy,omitempty"`
Status string `json:"status"`
ProposalCount int `json:"proposal_count"`
ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"`
ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy string `json:"replacement_policy,omitempty"`
Status string `json:"status"`
ProposalCount int `json:"proposal_count"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"`
ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
}
type ValidatorDecisionReport struct {

View File

@@ -47,7 +47,7 @@ func (h testChunkProposalHarness) collectEnrichedProposals(
return nil, err
}
for _, proposal := range base {
for _, proposal := range base.Proposals {
sectionIndex := section.Index
out = append(out, proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposal,
@@ -85,11 +85,11 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy
func (m deterministicFakeModule) Validators() []Validator { return nil }
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
_ = ctx
if req.WorkingTranscript == nil || req.Section == nil {
return []proposals.CorrectionProposal{}, nil
return ProposalResult{}, nil
}
out := make([]proposals.CorrectionProposal, 0)
@@ -110,7 +110,7 @@ func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalReques
}
}
return out, nil
return ProposalResult{Proposals: out}, nil
}
func TestChunkProposalMetadataAssociation(t *testing.T) {

View File

@@ -11,6 +11,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
// StructuredLLMClient provides provider-agnostic structured completion.
@@ -28,7 +29,7 @@ type TranscriptModule interface {
Key() string
ReplacementPolicy() proposals.ReplacementPolicy
Validators() []Validator
Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error)
Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error)
}
// Validator evaluates candidate proposals and returns one decision per proposal index.
@@ -103,6 +104,11 @@ type ProposalRequest struct {
LLMScheduler LLMScheduler `json:"-"`
}
type ProposalResult struct {
Proposals []proposals.CorrectionProposal `json:"proposals,omitempty"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
}
// ValidationRequest is the input to validator execution.
type ValidationRequest = validators.Request

View File

@@ -50,11 +50,13 @@ func (f *fakeModule) Validators() []Validator {
return []Validator{&fakeValidator{}}
}
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
_ = ctx
_ = req
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
return ProposalResult{
Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
},
}, nil
}
@@ -68,12 +70,12 @@ func TestInterfaceContractsCompileWithFakes(t *testing.T) {
t.Fatalf("unexpected module key: %q", got)
}
proposalsOut, err := module.Propose(context.Background(), ProposalRequest{})
proposalResult, err := module.Propose(context.Background(), ProposalRequest{})
if err != nil {
t.Fatalf("unexpected propose error: %v", err)
}
if len(proposalsOut) != 1 {
t.Fatalf("expected one proposal, got %d", len(proposalsOut))
if len(proposalResult.Proposals) != 1 {
t.Fatalf("expected one proposal, got %d", len(proposalResult.Proposals))
}
}

View File

@@ -19,10 +19,10 @@ func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m noopModule) Validators() []contracts.Validator { return nil }
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
_ = ctx
_ = req
return nil, nil
return contracts.ProposalResult{}, nil
}
func TestKnownModuleKeyRecognition(t *testing.T) {

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
@@ -68,6 +69,7 @@ type Request struct {
type Result struct {
Corrections []proposals.CorrectionProposal `json:"corrections"`
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
}
@@ -156,6 +158,12 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}
if callErr != nil {
if isMalformedStructuredOutputError(callErr) {
return Result{
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
Artifacts: artifacts,
}, nil
}
return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr)
}
@@ -168,9 +176,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
CorrectedText: raw.CorrectedText,
Confidence: raw.Confidence,
}
if err := candidate.Validate(); err != nil {
return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err)
}
corrections = append(corrections, candidate)
enrichedCandidate := proposals.EnrichedCorrectionProposal{
@@ -191,6 +196,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
return Result{
Corrections: corrections,
Enriched: enriched,
Warnings: nil,
Artifacts: artifacts,
}, nil
}
@@ -240,6 +246,48 @@ func errPayload(err error) any {
return map[string]any{"error": err.Error()}
}
func isMalformedStructuredOutputError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
warning := stagewarnings.StageWarning{
Scope: stagewarnings.ScopeProposalGeneration,
ReasonCode: "proposal_response_malformed",
Message: strings.TrimSpace(err.Error()),
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
}
if section != nil {
sectionIndex := section.Index
warning.SectionIndex = &sectionIndex
}
return warning
}
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
if artifacts.ErrorPayloadPath != "" {
return artifacts.ErrorPayloadPath
}
return artifacts.ResponsePayloadPath
}
type diagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter
}

View File

@@ -224,12 +224,12 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
}
}
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9},
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
},
},
},
@@ -237,9 +237,38 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
req := defaultRequest(t)
req.LLMClient = client
_, err := GenerateCandidates(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "invalid structured correction") {
t.Fatalf("expected structured response validation failure, got %v", err)
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected invalid correction to survive generation, got %v", err)
}
if len(result.Corrections) != 1 {
t.Fatalf("expected one correction, got %+v", result)
}
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
}
if len(result.Warnings) != 0 {
t.Fatalf("did not expect warnings for individually invalid corrections, got %+v", result.Warnings)
}
}
func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) {
client := &fakeStructuredClient{err: errors.New("malformed structured output")}
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed structured output to downgrade to warning, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Warnings) != 1 {
t.Fatalf("expected one warning, got %+v", result.Warnings)
}
if result.Warnings[0].ReasonCode != "proposal_response_malformed" {
t.Fatalf("unexpected warning: %+v", result.Warnings[0])
}
}

View File

@@ -147,6 +147,8 @@ func skipReasonMessage(reason ProposalSkipReason) string {
return "original_text matched multiple spans under require_unique policy"
case SkipReasonNoEffect:
return "original_text and corrected_text are identical"
case SkipReasonEmptyResultingText:
return "proposal would leave the segment empty"
case SkipReasonInvalidProposal:
return "proposal failed structural or policy validation"
default:

View File

@@ -15,6 +15,7 @@ const (
SkipReasonMissingOriginalText ProposalSkipReason = "missing_original_text"
SkipReasonAmbiguousOriginal ProposalSkipReason = "ambiguous_original_text"
SkipReasonNoEffect ProposalSkipReason = "no_effect"
SkipReasonEmptyResultingText ProposalSkipReason = "empty_resulting_segment"
SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal"
)
@@ -62,6 +63,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
}
corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
}
return SegmentPreviewResult{
Applicable: true,
CorrectedSegmentText: corrected,
@@ -70,6 +74,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
case ReplacementPolicyReplaceAll:
corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
}
return SegmentPreviewResult{
Applicable: true,
CorrectedSegmentText: corrected,

View File

@@ -121,6 +121,42 @@ func TestPreviewProposalForSegmentNoEffectReplacement(t *testing.T) {
}
}
func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonEmpty(t *testing.T) {
segment := &schema.Segment{ID: 8, Text: "uh hello"}
proposal := CorrectionProposal{
TargetSegmentID: 8,
OriginalText: "uh ",
CorrectedText: "",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
if !result.Applicable {
t.Fatalf("expected applicable preview, got skip reason %q", result.SkipReason)
}
if result.CorrectedSegmentText != "hello" {
t.Fatalf("unexpected corrected text: %q", result.CorrectedSegmentText)
}
}
func TestPreviewProposalForSegmentRejectsEmptyResultingSegment(t *testing.T) {
segment := &schema.Segment{ID: 8, Text: "uh"}
proposal := CorrectionProposal{
TargetSegmentID: 8,
OriginalText: "uh",
CorrectedText: "",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
if result.Applicable {
t.Fatal("expected non-applicable preview")
}
if result.SkipReason != SkipReasonEmptyResultingText {
t.Fatalf("expected skip reason %q, got %q", SkipReasonEmptyResultingText, result.SkipReason)
}
}
func TestPreviewProposalForSegmentPreservesInputSegment(t *testing.T) {
segment := &schema.Segment{ID: 9, Speaker: "A", Start: 1.0, End: 2.0, Text: "rank rank"}
original := *segment

View File

@@ -38,9 +38,6 @@ func (p CorrectionProposal) Validate() error {
if strings.TrimSpace(p.OriginalText) == "" {
return fmt.Errorf("proposal original_text must not be empty")
}
if strings.TrimSpace(p.CorrectedText) == "" {
return fmt.Errorf("proposal corrected_text must not be empty")
}
if p.Confidence < 0.0 || p.Confidence > 1.0 {
return fmt.Errorf("proposal confidence must be between 0.0 and 1.0")
}

View File

@@ -35,7 +35,7 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) {
}
}
func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
proposal := CorrectionProposal{
TargetSegmentID: 42,
OriginalText: "gestures",
@@ -43,12 +43,8 @@ func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
Confidence: 0.95,
}
err := proposal.Validate()
if err == nil {
t.Fatal("expected validation error, got nil")
}
if err.Error() != "proposal corrected_text must not be empty" {
t.Fatalf("unexpected error: %v", err)
if err := proposal.Validate(); err != nil {
t.Fatalf("expected empty corrected_text to be allowed, got %v", err)
}
}

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
@@ -37,18 +38,19 @@ type ValidationScheduler = contracts.LLMScheduler
// ModuleResult captures deterministic per-module execution output.
type ModuleResult struct {
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Status string `json:"status"`
ProposalCount int `json:"proposal_count"`
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Status string `json:"status"`
ProposalCount int `json:"proposal_count"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
}
type ValidatorDecisionRecord struct {
@@ -176,6 +178,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
ReplacementPolicy: policy,
Status: ModuleStatusFailed,
ProposalCount: pipelineResult.ProposalCount,
Warnings: pipelineResult.Warnings,
ValidatorDecisions: pipelineResult.ValidatorDecisions,
ValidatorRejected: pipelineResult.ValidatorRejected,
ErrorMessage: pipelineErr.Error(),
@@ -199,6 +202,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
ReplacementPolicy: policy,
Status: ModuleStatusSuccess,
ProposalCount: pipelineResult.ProposalCount,
Warnings: pipelineResult.Warnings,
ValidatorDecisions: pipelineResult.ValidatorDecisions,
ValidatorRejected: pipelineResult.ValidatorRejected,
AppliedChanges: applyResult.Applied,
@@ -232,12 +236,14 @@ type collectSectionProposalsInput struct {
type sectionProposals struct {
meta contracts.SectionMetadata
corrected []proposals.CorrectionProposal
warnings []stagewarnings.StageWarning
}
type sectionProposalResult struct {
sectionPos int
section chunking.Section
corrected []proposals.CorrectionProposal
warnings []stagewarnings.StageWarning
err error
}
@@ -247,12 +253,14 @@ type sectionValidationResult struct {
approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange
warnings []stagewarnings.StageWarning
err error
}
type modulePipelineResult struct {
ProposalCount int
Approved []proposals.EnrichedCorrectionProposal
Warnings []stagewarnings.StageWarning
ValidatorDecisions []ValidatorDecisionRecord
ValidatorRejected []ValidatorRejectedChange
}
@@ -271,7 +279,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
go func() {
defer wg.Done()
meta := contracts.SectionMetadataFromSection(section)
corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
proposalResult, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: input.Config,
WorkingTranscript: transcriptFromSection(section),
@@ -291,7 +299,8 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
case results <- sectionProposalResult{
sectionPos: sectionPos,
section: section,
corrected: corrected,
corrected: proposalResult.Proposals,
warnings: proposalResult.Warnings,
err: err,
}:
case <-runCtx.Done():
@@ -308,6 +317,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
out := modulePipelineResult{
Approved: make([]proposals.EnrichedCorrectionProposal, 0),
Warnings: make([]stagewarnings.StageWarning, 0),
ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
ValidatorRejected: make([]ValidatorRejectedChange, 0),
}
@@ -352,6 +362,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if firstErr != nil {
continue
}
out.Warnings = append(out.Warnings, result.warnings...)
pending[result.sectionPos] = result
for {
@@ -401,6 +412,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
approved: validated.approved,
decisions: validated.decisions,
rejected: validated.rejected,
warnings: validated.warnings,
err: err,
}
}(nextSectionToProcess, sectionEnriched, sectionMeta)
@@ -424,6 +436,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
break
}
out.Approved = append(out.Approved, res.approved...)
out.Warnings = append(out.Warnings, res.warnings...)
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
}
@@ -440,6 +453,38 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName]
})
sort.SliceStable(out.Warnings, func(i, j int) bool {
leftSection, rightSection := -1, -1
if out.Warnings[i].SectionIndex != nil {
leftSection = *out.Warnings[i].SectionIndex
}
if out.Warnings[j].SectionIndex != nil {
rightSection = *out.Warnings[j].SectionIndex
}
if leftSection != rightSection {
return leftSection < rightSection
}
leftBatch, rightBatch := -1, -1
if out.Warnings[i].BatchIndex != nil {
leftBatch = *out.Warnings[i].BatchIndex
}
if out.Warnings[j].BatchIndex != nil {
rightBatch = *out.Warnings[j].BatchIndex
}
if leftBatch != rightBatch {
return leftBatch < rightBatch
}
if out.Warnings[i].ValidatorName != out.Warnings[j].ValidatorName {
return validatorOrder[out.Warnings[i].ValidatorName] < validatorOrder[out.Warnings[j].ValidatorName]
}
if out.Warnings[i].Scope != out.Warnings[j].Scope {
return out.Warnings[i].Scope < out.Warnings[j].Scope
}
if out.Warnings[i].ReasonCode != out.Warnings[j].ReasonCode {
return out.Warnings[i].ReasonCode < out.Warnings[j].ReasonCode
}
return out.Warnings[i].Message < out.Warnings[j].Message
})
if firstErr != nil {
return out, firstErr
@@ -467,11 +512,13 @@ type validateSectionCandidatesResult struct {
approved []proposals.EnrichedCorrectionProposal
decisions []ValidatorDecisionRecord
rejected []ValidatorRejectedChange
warnings []stagewarnings.StageWarning
}
func validateSectionCandidates(ctx context.Context, input validateSectionCandidatesInput) (validateSectionCandidatesResult, error) {
decisions := make([]ValidatorDecisionRecord, 0)
rejected := make([]ValidatorRejectedChange, 0)
warnings := make([]stagewarnings.StageWarning, 0)
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
for _, validator := range input.Validators {
@@ -512,6 +559,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
approved: eligible,
decisions: decisions,
rejected: rejected,
warnings: warnings,
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
}
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
@@ -519,8 +567,10 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
approved: eligible,
decisions: decisions,
rejected: rejected,
warnings: warnings,
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err)
}
warnings = append(warnings, vResult.Warnings...)
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
@@ -562,6 +612,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
approved: eligible,
decisions: decisions,
rejected: rejected,
warnings: warnings,
}, nil
}

View File

@@ -47,11 +47,12 @@ type fakeModule struct {
func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if m.proposeF == nil {
return nil, nil
return contracts.ProposalResult{}, nil
}
return m.proposeF(req)
proposalsOut, err := m.proposeF(req)
return contracts.ProposalResult{Proposals: proposalsOut}, err
}
type fakeValidator struct {
@@ -1191,7 +1192,7 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}
}
func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) {
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1209,15 +1210,21 @@ func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
ValidationLLMClient: client,
})
if err == nil {
t.Fatal("expected llm validator failure")
if err != nil {
t.Fatalf("expected malformed validator response to downgrade, got %v", err)
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected partial progress retained")
}
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
}
if len(out.ModuleResults[1].Warnings) != 1 || out.ModuleResults[1].Warnings[0].ReasonCode != validators.ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", out.ModuleResults[1].Warnings)
}
}
func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1232,12 +1239,12 @@ func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
})
if err == nil {
t.Fatal("expected missing decision failure")
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
}
}
func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
@@ -1255,8 +1262,8 @@ func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
})
if err == nil {
t.Fatal("expected duplicate decision failure")
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
}
}
@@ -1355,7 +1362,7 @@ type proposalGenerationModule struct {
func (m proposalGenerationModule) Key() string { return m.key }
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
@@ -1372,9 +1379,9 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return nil, err
return contracts.ProposalResult{}, err
}
return result.Corrections, nil
return contracts.ProposalResult{Proposals: result.Corrections, Warnings: result.Warnings}, nil
}
type fakeProposalStructuredClient struct {

View File

@@ -4,8 +4,35 @@ import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
)
type ProposalShapeValidator struct{}
func (v ProposalShapeValidator) Name() string { return "proposal_shape" }
func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result, error) {
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
switch {
case c.TargetSegmentID <= 0:
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
case strings.TrimSpace(c.OriginalText) == "":
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
case c.Confidence < 0.0 || c.Confidence > 1.0:
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
default:
decisions = append(decisions, approval(c.ProposalIndex))
}
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type ConfidenceThresholdValidator struct{}
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
@@ -62,10 +89,23 @@ type NonEmptyCorrectionValidator struct{}
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" }
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
segmentsByID := make(map[int]schema.Segment)
if req.WorkingTranscript != nil {
for _, seg := range req.WorkingTranscript.Segments {
segmentsByID[seg.ID] = seg
}
}
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
if strings.TrimSpace(c.CorrectedText) == "" {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty"))
segment, ok := segmentsByID[c.TargetSegmentID]
if !ok {
decisions = append(decisions, approval(c.ProposalIndex))
continue
}
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy)
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))

View File

@@ -11,6 +11,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
@@ -78,7 +79,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
maxTokens = req.Config.ValidationMaxPromptTokens
}
batches, err := ChunkLLMValidationItems(validationReq.Items, maxTokens, v.estimator)
warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
if len(itemsForBatching) == 0 {
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
}
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
if err != nil {
return Result{}, err
}
@@ -140,12 +154,19 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
)
}
if err != nil {
if isMalformedStructuredOutputError(err) {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
continue
}
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
}
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
if err != nil {
return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err)
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
continue
}
for i := range batchDecisions {
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
@@ -154,12 +175,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
all = append(all, llmDecisions...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all}, nil
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
}
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
@@ -301,3 +323,98 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
}
return decisions, nil
}
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
out := make([]Decision, 0)
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
}
return out
}
func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
out := make([]stagewarnings.StageWarning, 0)
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
ReasonCode: ReasonValidatorInputTooLarge,
Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
})
}
return out
}
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
if len(decisions) == 0 {
return append([]LLMValidationItem(nil), items...)
}
rejected := make(map[int]struct{}, len(decisions))
for _, decision := range decisions {
rejected[decision.ProposalIndex] = struct{}{}
}
out := make([]LLMValidationItem, 0, len(items))
for _, item := range items {
if _, ok := rejected[item.CorrectionIndex]; ok {
continue
}
out = append(out, item)
}
return out
}
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
out := make([]Decision, 0, len(items))
for _, item := range items {
out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
}
return out
}
func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
idx := batchIndex
return stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
BatchIndex: &idx,
ReasonCode: reasonCode,
Message: strings.TrimSpace(message),
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
}
}
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
if artifacts.ErrorPayloadPath != "" {
return artifacts.ErrorPayloadPath
}
return artifacts.ResponsePayloadPath
}
func isMalformedStructuredOutputError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}

View File

@@ -248,29 +248,38 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
}
}
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "completion failed") {
t.Fatalf("expected malformed output error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed output downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
}
}
func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) {
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "response invalid") {
t.Fatalf("expected missing decision error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
@@ -278,20 +287,74 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "duplicate") {
t.Fatalf("expected duplicate decision error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) {
func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "unknown") {
t.Fatalf("expected unknown index error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected unknown index downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
huge := strings.Repeat("gestures ", 200)
req := Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Text: huge},
{ID: 2, Text: "There were gestures at the temple.", Categories: []string{"narration"}},
}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
},
ModuleKey: "homophones",
ModuleInstance: "homophones",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
}
req.LLMClient = client
cfg := config.Default()
cfg.ValidationMaxPromptTokens = 200
req.Config = &cfg
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected oversize downgrade, got %v", err)
}
if len(res.Decisions) != 2 {
t.Fatalf("expected two decisions, got %+v", res.Decisions)
}
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved {
t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
}
if !res.Decisions[1].Approved {
t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("expected one oversize warning, got %+v", res.Warnings)
}
}

View File

@@ -8,16 +8,22 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
const (
ReasonApproved = "approved"
ReasonLowConfidence = "low_confidence"
ReasonMissingOriginalText = "missing_original_text"
ReasonMissingTargetSegment = "missing_target_segment"
ReasonEmptyCorrectedText = "empty_corrected_text"
ReasonNoEffect = "no_effect"
ReasonProtectedGlossaryTerm = "protected_glossary_term"
ReasonApproved = "approved"
ReasonLowConfidence = "low_confidence"
ReasonMissingOriginalText = "missing_original_text"
ReasonMissingTargetSegment = "missing_target_segment"
ReasonEmptyResultingText = "empty_resulting_segment"
ReasonNoEffect = "no_effect"
ReasonProtectedGlossaryTerm = "protected_glossary_term"
ReasonInvalidTargetSegment = "invalid_target_segment_id"
ReasonEmptyOriginalText = "empty_original_text"
ReasonInvalidConfidence = "invalid_confidence"
ReasonValidatorMalformed = "validator_response_malformed"
ReasonValidatorInputTooLarge = "validator_input_too_large"
)
// Request is the runtime input shared by deterministic validators.
@@ -45,8 +51,9 @@ type Decision struct {
// Result is one validator output containing exactly one decision per proposal index.
type Result struct {
ValidatorName string `json:"validator_name"`
Decisions []Decision `json:"decisions"`
ValidatorName string `json:"validator_name"`
Decisions []Decision `json:"decisions"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
}
// ValidationScheduler provides bounded execution for validator LLM calls.

View File

@@ -68,6 +68,31 @@ func TestConfidenceThresholdValidator(t *testing.T) {
}
}
func TestProposalShapeValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 0, "teh", "the", 0.9),
mkCandidate(2, 1, " ", "the", 0.9),
mkCandidate(3, 1, "teh", "the", 1.5),
}}
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved {
t.Fatalf("expected proposal 0 approved")
}
if res.Decisions[1].ReasonCode != ReasonInvalidTargetSegment {
t.Fatalf("expected invalid target segment rejection, got %+v", res.Decisions[1])
}
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
}
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
}
}
func TestOriginalTextPresenceValidator(t *testing.T) {
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
@@ -90,10 +115,13 @@ func TestOriginalTextPresenceValidator(t *testing.T) {
}
func TestNonEmptyCorrectionValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "hello", " ", 0.9),
}}
req := Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "hello world", " ", 0.9),
}}
res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
@@ -101,8 +129,8 @@ func TestNonEmptyCorrectionValidator(t *testing.T) {
if !res.Decisions[0].Approved {
t.Fatalf("expected proposal 0 approved")
}
if res.Decisions[1].ReasonCode != ReasonEmptyCorrectedText {
t.Fatalf("expected empty_corrected_text, got %+v", res.Decisions[1])
if res.Decisions[1].ReasonCode != ReasonEmptyResultingText {
t.Fatalf("expected empty_resulting_segment, got %+v", res.Decisions[1])
}
}
@@ -151,9 +179,14 @@ func TestStableReasonCodes(t *testing.T) {
ReasonLowConfidence,
ReasonMissingOriginalText,
ReasonMissingTargetSegment,
ReasonEmptyCorrectedText,
ReasonEmptyResultingText,
ReasonNoEffect,
ReasonProtectedGlossaryTerm,
ReasonInvalidTargetSegment,
ReasonEmptyOriginalText,
ReasonInvalidConfidence,
ReasonValidatorMalformed,
ReasonValidatorInputTooLarge,
}
for _, code := range codes {
if strings.TrimSpace(code) == "" {

View File

@@ -0,0 +1,18 @@
package warnings
type Scope string
const (
ScopeProposalGeneration Scope = "proposal_generation"
ScopeValidator Scope = "validator"
)
type StageWarning struct {
Scope Scope `json:"scope"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
SectionIndex *int `json:"section_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
BatchIndex *int `json:"batch_index,omitempty"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}

View File

@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0
if req.Section != nil {
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
return contracts.ProposalResult{}, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return nil, err
return contracts.ProposalResult{}, err
}
return generated.Corrections, nil
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
}
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {

View File

@@ -129,6 +129,7 @@ func TestGlossaryModuleValidatorChain(t *testing.T) {
got = append(got, v.Name())
}
want := []string{
"proposal_shape",
"no_effect",
"original_text_presence",
"confidence_threshold",
@@ -185,7 +186,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
if len(client.calls) != 1 || client.calls[0].StageName != "glossary:proposal" {
t.Fatalf("expected one glossary:proposal call, got %+v", client.calls)
}
if len(out) != 1 || out[0].CorrectedText != "Jesters" {
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
t.Fatalf("unexpected proposals: %+v", out)
}
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "glossary", "*proposal*response-payload.json"))

View File

@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0
if req.Section != nil {
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
return contracts.ProposalResult{}, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return nil, err
return contracts.ProposalResult{}, err
}
return generated.Corrections, nil
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
}
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {

View File

@@ -131,6 +131,7 @@ func TestGrammarModuleValidatorChain(t *testing.T) {
got = append(got, v.Name())
}
want := []string{
"proposal_shape",
"no_effect",
"original_text_presence",
"confidence_threshold",
@@ -187,7 +188,7 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal" {
t.Fatalf("expected one grammar:proposal call, got %+v", client.calls)
}
if len(out) != 1 || out[0].CorrectedText != "Hello, world" {
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Hello, world" {
t.Fatalf("unexpected proposals: %+v", out)
}
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "grammar", "*proposal*response-payload.json"))

View File

@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0
if req.Section != nil {
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
return contracts.ProposalResult{}, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return nil, err
return contracts.ProposalResult{}, err
}
return generated.Corrections, nil
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
}
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {

View File

@@ -145,6 +145,7 @@ func TestHomophonesModuleValidatorChain(t *testing.T) {
got = append(got, v.Name())
}
want := []string{
"proposal_shape",
"no_effect",
"original_text_presence",
"confidence_threshold",
@@ -201,7 +202,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" {
t.Fatalf("expected one homophones:proposal call, got %+v", client.calls)
}
if len(out) != 1 || out[0].CorrectedText != "Jesters" {
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
t.Fatalf("unexpected proposals: %+v", out)
}
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json"))

View File

@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
sectionIndex := 0
if req.Section != nil {
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
return contracts.ProposalResult{}, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return nil, err
return contracts.ProposalResult{}, err
}
return generated.Corrections, nil
return contracts.ProposalResult{
Proposals: generated.Corrections,
Warnings: generated.Warnings,
}, nil
}
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {

View File

@@ -132,6 +132,7 @@ func TestSpokenWordModuleValidatorChain(t *testing.T) {
got = append(got, v.Name())
}
want := []string{
"proposal_shape",
"no_effect",
"original_text_presence",
"confidence_threshold",
@@ -188,7 +189,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
if len(client.calls) != 1 || client.calls[0].StageName != "spoken_word:proposal" {
t.Fatalf("expected one spoken_word:proposal call, got %+v", client.calls)
}
if len(out) != 1 || out[0].CorrectedText != "I think" {
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "I think" {
t.Fatalf("unexpected proposals: %+v", out)
}
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "spoken_word", "*proposal*response-payload.json"))

View File

@@ -9,6 +9,7 @@ import (
var builtInChains = map[string][]string{
"glossary": {
KeyProposalShape,
KeyNoEffect,
KeyOriginalTextPresence,
KeyConfidenceThreshold,
@@ -18,6 +19,7 @@ var builtInChains = map[string][]string{
KeyMeaningReversalReview,
},
"homophones": {
KeyProposalShape,
KeyNoEffect,
KeyOriginalTextPresence,
KeyConfidenceThreshold,
@@ -27,6 +29,7 @@ var builtInChains = map[string][]string{
KeyMeaningReversalReview,
},
"spoken_word": {
KeyProposalShape,
KeyNoEffect,
KeyOriginalTextPresence,
KeyConfidenceThreshold,
@@ -36,6 +39,7 @@ var builtInChains = map[string][]string{
KeyMeaningReversalReview,
},
"grammar": {
KeyProposalShape,
KeyNoEffect,
KeyOriginalTextPresence,
KeyConfidenceThreshold,

View File

@@ -0,0 +1,13 @@
package proposal_shape
import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
const Key = "proposal_shape"
func New() (contracts.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.ProposalShapeValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -11,11 +11,13 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
"gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape"
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
"gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility"
)
const (
KeyProposalShape = "proposal_shape"
KeyConfidenceThreshold = "confidence_threshold"
KeyOriginalTextPresence = "original_text_presence"
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
@@ -39,6 +41,7 @@ type Registry struct {
func NewBuiltInRegistry() *Registry {
defs := []BuiltInValidatorDefinition{
{Key: KeyProposalShape, Build: proposal_shape.New},
{Key: KeyConfidenceThreshold, Build: confidence_threshold.New},
{Key: KeyOriginalTextPresence, Build: original_text_presence.New},
{Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New},

View File

@@ -14,6 +14,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
"gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape"
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
"gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility"
)
@@ -21,6 +22,7 @@ import (
func TestBuiltInRegistryRegistersAllKeys(t *testing.T) {
r := NewBuiltInRegistry()
for _, key := range []string{
KeyProposalShape,
KeyConfidenceThreshold,
KeyOriginalTextPresence,
KeyNonEmptyCorrectedText,
@@ -52,6 +54,7 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
wantClass validatormetadata.ExecutionClass
}
cases := []validatorCtor{
{name: "proposal_shape", key: KeyProposalShape, build: proposal_shape.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "confidence_threshold", key: KeyConfidenceThreshold, build: confidence_threshold.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "non_empty_corrected_text", key: KeyNonEmptyCorrectedText, build: non_empty_corrected_text.New, wantClass: validatormetadata.ExecutionClassDeterministic},