Make module-stage LLM handling resilient and report warnings
This commit is contained in:
10
README.md
10
README.md
@@ -19,6 +19,7 @@ Pipeline behavior includes:
|
||||
- conservative spoken-word dysfluency cleanup with semantic guardrails
|
||||
- grammar/punctuation/capitalization/formatting cleanup
|
||||
- validator-chain enforcement before application
|
||||
- malformed module-stage LLM payloads degrade to warnings/rejections instead of failing the run
|
||||
- run reports and diagnostics artifacts with secret redaction
|
||||
|
||||
## Build and Install
|
||||
@@ -130,6 +131,7 @@ audita process transcript.json \
|
||||
- Without `--output`, stdout contains transcript JSON only on success.
|
||||
- `--report-json` writes a file and is never printed to stdout.
|
||||
- stderr is human-readable diagnostics/errors.
|
||||
- successful runs remain quiet on stderr even when module warnings are recorded in report/diagnostics artifacts.
|
||||
|
||||
For subprocess orchestration guidance, see [`docs/subprocess-operations.md`](docs/subprocess-operations.md).
|
||||
|
||||
@@ -149,10 +151,10 @@ audita config print-effective --config audita.yml
|
||||
```
|
||||
|
||||
For full config-file schema and examples, see [`docs/configuration.md`](docs/configuration.md).
|
||||
For output-schema details, see [`docs/output-schemas.md`](docs/output-schemas.md).
|
||||
For built-in validator keys and chain definitions, see [`docs/validators.md`](docs/validators.md).
|
||||
For embedded prompt assets and prompt metadata behavior, see [`docs/prompts.md`](docs/prompts.md).
|
||||
For CLI/process compatibility guarantees, see [`docs/public-contract.md`](docs/public-contract.md).
|
||||
For output-schema details, see [`docs/architecture/output-schemas.md`](docs/architecture/output-schemas.md).
|
||||
For built-in validator keys and chain definitions, see [`docs/architecture/validators.md`](docs/architecture/validators.md).
|
||||
For embedded prompt assets and prompt metadata behavior, see [`docs/architecture/prompts.md`](docs/architecture/prompts.md).
|
||||
For CLI/process compatibility guarantees, see [`docs/architecture/public-contract.md`](docs/architecture/public-contract.md).
|
||||
|
||||
### Modules
|
||||
|
||||
|
||||
@@ -382,25 +382,22 @@ func TestProcessFailureMalformedStructuredLLMResponseViaSubprocessHook(t *testin
|
||||
"--work-dir-retention",
|
||||
"always",
|
||||
)
|
||||
if result.exitCode == 0 {
|
||||
t.Fatalf("expected nonzero exit code")
|
||||
if result.exitCode != 0 {
|
||||
t.Fatalf("expected zero exit code, got %d stderr=%q", result.exitCode, result.stderr)
|
||||
}
|
||||
if result.stdout != "" {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", result.stdout)
|
||||
if !json.Valid([]byte(result.stdout)) {
|
||||
t.Fatalf("expected transcript JSON on stdout, got %q", result.stdout)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "runner_execution") {
|
||||
t.Fatalf("expected runner_execution failure, got %q", result.stderr)
|
||||
}
|
||||
if !strings.Contains(result.stderr, "diagnostics:") {
|
||||
t.Fatalf("expected diagnostics path in stderr, got %q", result.stderr)
|
||||
if result.stderr != "" {
|
||||
t.Fatalf("expected empty stderr on success, got %q", result.stderr)
|
||||
}
|
||||
report := readFile(t, reportPath)
|
||||
if !json.Valid(report) {
|
||||
t.Fatalf("expected valid failure report JSON")
|
||||
t.Fatalf("expected valid success report JSON")
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log, got: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -133,11 +133,16 @@ internal/framework/validators/
|
||||
llm_batching.go
|
||||
llm_validators.go
|
||||
|
||||
internal/framework/warnings/
|
||||
warnings.go
|
||||
|
||||
internal/validators/
|
||||
metadata/
|
||||
metadata.go
|
||||
registry.go
|
||||
chains.go
|
||||
proposal_shape/
|
||||
validator.go
|
||||
confidence_threshold/
|
||||
validator.go
|
||||
original_text_presence/
|
||||
@@ -242,6 +247,7 @@ Important behavior details:
|
||||
- Explicit `--modules grammar`, `--modules glossary`, `--modules homophones`, and `--modules spoken_word` continue to run production module paths with LLM-backed proposal generation and validator-chain execution.
|
||||
- Default runs (without explicit module selection) perform LLM calls through production module and validator paths.
|
||||
- Success path is generally quiet on stderr.
|
||||
- Malformed module-stage LLM payloads degrade to validator rejections and module warnings instead of aborting the run.
|
||||
- Source IDs are preserved into a canonical transcript before normalization; normalization then reassigns output IDs sequentially from `1`.
|
||||
|
||||
## Implemented data contracts
|
||||
@@ -539,20 +545,20 @@ Validator execution classification metadata:
|
||||
|
||||
Stable built-in validator keys:
|
||||
- deterministic:
|
||||
- `proposal_shape`
|
||||
- `confidence_threshold`
|
||||
- `original_text_presence`
|
||||
- `non_empty_corrected_text`
|
||||
- `non_empty_corrected_text` (historical key name; current semantics reject empty resulting segment text)
|
||||
- `no_effect`
|
||||
- `protected_terms`
|
||||
- LLM-backed:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `editorial_review`
|
||||
- `grammar_review`
|
||||
- `spoken_word_review`
|
||||
|
||||
Built-in module chains:
|
||||
- `glossary`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -561,6 +567,7 @@ Built-in module chains:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `homophones`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -569,20 +576,22 @@ Built-in module chains:
|
||||
- `spoken_form_plausibility`
|
||||
- `meaning_reversal_review`
|
||||
- `spoken_word`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `spoken_word_review`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
- `grammar`:
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
- `protected_terms`
|
||||
- `non_empty_corrected_text`
|
||||
- `grammar_review`
|
||||
- `editorial_review`
|
||||
- `meaning_reversal_review`
|
||||
|
||||
1.0 boundary:
|
||||
@@ -595,12 +604,14 @@ Built-in module chains:
|
||||
- prompt builders for:
|
||||
- spoken-form plausibility
|
||||
- meaning reversal detection
|
||||
- editorial review
|
||||
- grammar review
|
||||
- spoken-word review
|
||||
- editorial review
|
||||
- grammar review
|
||||
- spoken-word review
|
||||
- deterministic batching by `validation_max_prompt_tokens`;
|
||||
- strict cardinality validation of structured LLM decisions (missing/duplicate/unknown indexes fail);
|
||||
- safe failure behavior for malformed/invalid structured responses.
|
||||
- strict cardinality validation of synthesized validator decision sets;
|
||||
- malformed validator payloads reject only the affected batch with warnings;
|
||||
- oversized single-proposal validator inputs reject only the affected proposal;
|
||||
- transport/provider/runtime LLM call failures remain fatal.
|
||||
|
||||
`internal/framework/runner` wires LLM validators into existing validator chains using:
|
||||
- the internal structured LLM client abstraction (`contracts.StructuredLLMClient`);
|
||||
@@ -726,6 +737,7 @@ Current process reports include diagnostics metadata references for:
|
||||
Current process reports also include:
|
||||
- module-level results (when runner modules execute), including applied/skipped proposal changes;
|
||||
- run-level module summary totals and failed module instance metadata.
|
||||
- module-level warning records for malformed proposal-generation payloads and malformed validator batches.
|
||||
- module-level validator decisions and validator rejections.
|
||||
- optional decision-level diagnostic artifact paths for validator LLM interactions when available.
|
||||
- stable validator keys in `validator_name` fields for validator decisions/rejections.
|
||||
|
||||
@@ -63,6 +63,7 @@ Ledger records are flattened review entries derived from module results and incl
|
||||
- deterministic and LLM validator decision snapshots using stable validator keys.
|
||||
|
||||
Validator rejection and proposal-application skip are distinct dispositions.
|
||||
Module warnings are reported in module results and diagnostics metadata, but do not create standalone correction-ledger rows.
|
||||
|
||||
## Report references
|
||||
|
||||
@@ -71,6 +72,8 @@ Validator rejection and proposal-application skip are distinct dispositions.
|
||||
- correction ledger artifact;
|
||||
- existing transcript/normalization/chunking/invocation/effective-config artifacts.
|
||||
|
||||
Module report entries also include warning records for malformed proposal-generation payloads and malformed validator batches.
|
||||
|
||||
## Retention behavior
|
||||
|
||||
Run-directory retention follows configured policy:
|
||||
@@ -93,6 +96,9 @@ When debugging:
|
||||
- validator rejections:
|
||||
- inspect `correction-ledger.json` rejected entries and matching validator decisions;
|
||||
- inspect validator response diagnostics payloads.
|
||||
- module warnings:
|
||||
- inspect module `warnings` entries in `report.json` or `--report-json`;
|
||||
- follow any diagnostic artifact path on the warning to the recorded error/response payload.
|
||||
- application skips:
|
||||
- inspect `correction-ledger.json` skipped entries and skip reason codes;
|
||||
- compare with validator decisions to distinguish validation rejection vs apply-time skip.
|
||||
|
||||
@@ -93,6 +93,7 @@ Current values:
|
||||
`--report-json` output and diagnostics run-dir `report.json` use the same report schema metadata.
|
||||
|
||||
Validator decision/rejection records in reports use stable validator keys in `validator_name`.
|
||||
Module results may also include warning records for malformed module-stage LLM payloads.
|
||||
Report diagnostics metadata includes artifact-path fields for utilization diagnostics and correction ledger when diagnostics initialization succeeds.
|
||||
|
||||
## Diagnostics directory behavior
|
||||
@@ -123,6 +124,7 @@ Success behavior:
|
||||
- with `--output`, stdout is empty
|
||||
- without `--output`, stdout contains only transcript JSON in selected output schema
|
||||
- report JSON is not written to stdout
|
||||
- success stderr remains empty even when reports/diagnostics contain module warnings
|
||||
|
||||
Failure behavior:
|
||||
- stderr contains human-readable error summary
|
||||
@@ -149,6 +151,7 @@ Config files should reference secrets via environment variable names (`api_key_e
|
||||
## Compatibility and deprecation policy
|
||||
|
||||
- Existing stable schema names, report metadata keys, and top-level command behavior are treated as public contract.
|
||||
- Existing stable validator keys remain public contract values even when validator semantics are refined.
|
||||
- Compatibility inputs (legacy flags/env aliases) may remain during transition windows.
|
||||
- Any planned removal or behavior change should include clear compatibility notes and migration guidance.
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ For LLM-backed validator prompt asset details, see [`docs/prompts.md`](prompts.m
|
||||
|
||||
Built-in validator construction is package-owned under `internal/validators/<validator_key>`:
|
||||
- `internal/validators/confidence_threshold`
|
||||
- `internal/validators/proposal_shape`
|
||||
- `internal/validators/original_text_presence`
|
||||
- `internal/validators/non_empty_corrected_text`
|
||||
- `internal/validators/no_effect`
|
||||
@@ -45,12 +46,14 @@ Current 1.0 boundary:
|
||||
|
||||
### Deterministic validators
|
||||
|
||||
- `proposal_shape`
|
||||
- rejects malformed proposal fields before other validators run.
|
||||
- `confidence_threshold`
|
||||
- checks proposal confidence against module-specific configured threshold.
|
||||
- `original_text_presence`
|
||||
- ensures target segment exists and `original_text` exists in current working segment text.
|
||||
- `non_empty_corrected_text`
|
||||
- rejects blank/whitespace-only `corrected_text`.
|
||||
- rejects proposals whose previewed resulting segment text would be empty or whitespace-only.
|
||||
- `no_effect`
|
||||
- rejects proposals where `original_text == corrected_text`.
|
||||
- `protected_terms`
|
||||
@@ -71,6 +74,7 @@ Current 1.0 boundary:
|
||||
Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
|
||||
- `glossary`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -80,6 +84,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `homophones`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -89,6 +94,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `spoken_word`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -98,6 +104,7 @@ Current built-in chains resolved from `internal/validators/chains.go`:
|
||||
- `meaning_reversal_review`
|
||||
|
||||
- `grammar`
|
||||
- `proposal_shape`
|
||||
- `no_effect`
|
||||
- `original_text_presence`
|
||||
- `confidence_threshold`
|
||||
@@ -119,7 +126,9 @@ Both variants preserve existing behavior and report the stable key `protected_te
|
||||
- modules execute serially;
|
||||
- section proposal work can run concurrently within a module;
|
||||
- deterministic validators run before LLM-backed validators;
|
||||
- malformed/missing/duplicate/unknown LLM validator decisions fail safely;
|
||||
- malformed module proposal payloads are downgraded to section-scoped module warnings with zero proposals for the affected section rather than module failure;
|
||||
- malformed/missing/duplicate/unknown LLM validator decisions reject the affected validator batch with warnings instead of failing the module;
|
||||
- oversized single-proposal validator inputs reject only the affected proposal under that validator;
|
||||
- approved proposals are applied once per module after section work settles.
|
||||
|
||||
## Validator rejections vs proposal-application skips
|
||||
@@ -128,12 +137,15 @@ Both variants preserve existing behavior and report the stable key `protected_te
|
||||
- proposal is denied by validator-chain review and appears in validator rejection reporting with validator key and reason code.
|
||||
- proposal-application skip:
|
||||
- proposal passed validators but could not be applied under replacement-policy semantics (for example no matching span at apply time).
|
||||
- module warning:
|
||||
- malformed proposal-generation payloads and malformed validator batches are recorded in module warning records and diagnostics without writing success stderr.
|
||||
|
||||
These are separate outcomes and are reported separately.
|
||||
|
||||
## Reporting and diagnostics identity
|
||||
|
||||
- report validator decision/rejection entries use stable validator keys in `validator_name`.
|
||||
- report module results include warning records for malformed module-stage LLM payloads.
|
||||
- validator LLM diagnostics include validator identity in interaction metadata and structured response schema metadata.
|
||||
- correction ledger entries include deterministic and LLM validator decision snapshots keyed by the same stable validator keys, and keep validator rejection distinct from application-level skip.
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
|
||||
- Verify structured response schemas are attached via `response_format.type=json_schema`.
|
||||
- Verify diagnostics metadata includes structured schema `id/version/name/sha256`.
|
||||
- Verify provider output is still locally decoded/validated before use.
|
||||
- Verify malformed module-stage structured payloads degrade to warnings/rejections instead of failing the run.
|
||||
|
||||
## Report and diagnostics schema checks
|
||||
|
||||
@@ -67,6 +68,7 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
|
||||
- Verify prompt metadata appears in LLM request metadata diagnostics:
|
||||
- `prompt_id`, `prompt_version`, `prompt_source`, `embedded_path`, `sha256`.
|
||||
- Verify stable validator keys appear in report decisions/rejections.
|
||||
- Verify module warning records appear in reports for malformed proposal-generation payloads and malformed validator batches.
|
||||
- Verify built-in validator chains resolve and execute for default and explicit module runs.
|
||||
|
||||
## Utilization diagnostics checks
|
||||
@@ -96,6 +98,8 @@ Use this checklist before cutting a pre-1.0 or 1.0 release candidate.
|
||||
## Failure and cancellation checks
|
||||
|
||||
- Verify controlled failure paths retain diagnostics and produce best-effort failure reports.
|
||||
- Verify malformed proposal-generation payloads keep exit code `0`, keep stderr empty on success, and record warnings in reports/diagnostics.
|
||||
- Verify malformed validator payloads reject only the affected batch and do not fail the module.
|
||||
- Verify timeout/cancellation paths exit nonzero, do not hang, and retain failure diagnostics when initialized.
|
||||
|
||||
## Release fixture/idempotence checks
|
||||
|
||||
72
docs/roadmap/publish.md
Normal file
72
docs/roadmap/publish.md
Normal file
@@ -0,0 +1,72 @@
|
||||
# Hard-Cutover Roadmap for Module-Stage LLM Resilience
|
||||
|
||||
## Summary
|
||||
|
||||
This roadmap captures the module-stage resilience work for Audita:
|
||||
|
||||
- fail fast on initialization, configuration, schema, and other pre-module setup errors;
|
||||
- remain resilient during module execution when LLM payloads are malformed or individual proposed corrections are invalid;
|
||||
- reject bad corrections through validator/reporting paths instead of aborting the module or process;
|
||||
- keep success stderr quiet; surface warnings only through report and diagnostics artifacts;
|
||||
- use a hard cutover only, with no compatibility aliases or transitional code.
|
||||
|
||||
Locked decisions:
|
||||
|
||||
- keep the stable public validator key `non_empty_corrected_text`;
|
||||
- change that validator’s behavior to mean “the resulting segment text must not be empty/whitespace-only after applying the proposal preview”;
|
||||
- malformed LLM-validator batch payloads reject the entire affected batch under that validator and continue;
|
||||
- proposal/validator transport failures, timeouts, and provider/runtime call failures remain fatal;
|
||||
- malformed structured payloads are downgraded; non-malformed runtime call failures are not.
|
||||
|
||||
## Stage 1: Proposal Intake Hardening
|
||||
|
||||
- Stop treating invalid individual structured corrections as fatal during proposal generation.
|
||||
- Preserve returned correction ordering and proposal-index assignment even when individual corrections are malformed.
|
||||
- Allow `corrected_text == ""` when the resulting segment remains non-empty after previewed application.
|
||||
- Downgrade malformed proposal-generation structured payloads into section-scoped warnings with zero proposals for that section.
|
||||
- Keep proposal-generation transport/provider/runtime call failures fatal.
|
||||
|
||||
Deterministic validation changes:
|
||||
|
||||
- Add `proposal_shape` as a built-in deterministic validator and run it first in every built-in module chain.
|
||||
- Reject malformed proposal fields with stable reason codes:
|
||||
- `invalid_target_segment_id`
|
||||
- `empty_original_text`
|
||||
- `invalid_confidence`
|
||||
- Keep `non_empty_corrected_text` as the stable validator key, but change its semantics to reject only `empty_resulting_segment`.
|
||||
- Keep validator rejection and apply-time skip as distinct outcomes.
|
||||
|
||||
## Stage 2: LLM Validator Resilience
|
||||
|
||||
- Keep validator transport/provider/runtime call failures fatal.
|
||||
- Downgrade malformed validator structured payloads into batch-scoped validator rejections plus module warnings.
|
||||
- Downgrade oversized single-proposal validator inputs into per-proposal validator rejections plus module warnings.
|
||||
- Preserve decision-cardinality enforcement as an internal invariant after malformed-payload degradation has synthesized complete decision sets.
|
||||
|
||||
Stable reason codes introduced or relied upon by this cutover:
|
||||
|
||||
- `empty_resulting_segment`
|
||||
- `invalid_target_segment_id`
|
||||
- `empty_original_text`
|
||||
- `invalid_confidence`
|
||||
- `validator_response_malformed`
|
||||
- `validator_input_too_large`
|
||||
- `proposal_response_malformed`
|
||||
|
||||
## Stage 3: Reporting, Diagnostics, and Docs
|
||||
|
||||
- Add module warning records to runner results and process reports.
|
||||
- Keep correction-ledger entries per-correction only; do not add standalone warning rows.
|
||||
- Record malformed proposal-generation and validator-batch warnings through report and diagnostics artifacts only.
|
||||
- Keep successful runs quiet on stderr even when warnings are present.
|
||||
- Update README and architecture/public-contract/diagnostics/validators/release-checklist docs to reflect the new behavior.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- Empty `corrected_text` may delete words, but proposals that would blank the whole segment are rejected or skipped safely.
|
||||
- Invalid proposal shape is rejected by validators, not by proposal generation.
|
||||
- Malformed proposal-generation payloads succeed with warnings and zero proposals for the affected section.
|
||||
- Malformed validator payloads reject only the affected validator batch and do not fail the module.
|
||||
- Oversized single validator inputs reject only the affected proposal.
|
||||
- Transport/provider/runtime LLM failures still fail the module and process.
|
||||
- Successful runs with warnings still exit `0`, emit transcript output normally, keep stderr empty, and expose warnings in report/diagnostics artifacts.
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
type noOpStructuredLLMClient struct{}
|
||||
@@ -803,6 +804,7 @@ func buildModuleReporting(runOutput *runner.RunOutput) (*reporting.ModulesSummar
|
||||
ReplacementPolicy: string(r.ReplacementPolicy),
|
||||
Status: r.Status,
|
||||
ProposalCount: r.ProposalCount,
|
||||
Warnings: append([]stagewarnings.StageWarning(nil), r.Warnings...),
|
||||
ValidatorDecisions: mapValidatorDecisions(r.ValidatorDecisions),
|
||||
ValidatorRejected: mapValidatorRejected(r.ValidatorRejected),
|
||||
AppliedChanges: r.AppliedChanges,
|
||||
|
||||
@@ -1793,11 +1793,12 @@ type fakeModule struct {
|
||||
func (m fakeModule) Key() string { return m.key }
|
||||
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if m.proposeF == nil {
|
||||
return nil, nil
|
||||
return contracts.ProposalResult{}, nil
|
||||
}
|
||||
return m.proposeF(req)
|
||||
proposalsOut, err := m.proposeF(req)
|
||||
return contracts.ProposalResult{Proposals: proposalsOut}, err
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
@@ -2384,7 +2385,7 @@ func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
|
||||
func TestRunProcessExplicitGrammarMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
t.Cleanup(func() { processProposalLLMClient = nil })
|
||||
|
||||
@@ -2403,22 +2404,32 @@ func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected failure")
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
||||
t.Fatalf("expected runner_execution error, got %q", stderr.String())
|
||||
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("expected transcript stdout on success: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
|
||||
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log on failed grammar run: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log on successful grammar run: %v", err)
|
||||
}
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
if report.Status != "success" || report.ErrorPhase != "" {
|
||||
t.Fatalf("expected successful report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
|
||||
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
|
||||
}
|
||||
if report.ModuleResults[0].Warnings[0].ReasonCode != "proposal_response_malformed" {
|
||||
t.Fatalf("unexpected warning: %+v", report.ModuleResults[0].Warnings[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3041,7 +3052,7 @@ func TestRunProcessExplicitGlossaryRepeatedStagesUseDeterministicInstanceNamesAn
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
|
||||
func TestRunProcessExplicitGlossaryMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
t.Cleanup(func() { processProposalLLMClient = nil })
|
||||
|
||||
@@ -3060,22 +3071,29 @@ func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testin
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected failure")
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
||||
t.Fatalf("expected runner_execution error, got %q", stderr.String())
|
||||
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("expected transcript stdout on success: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
|
||||
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log on failed glossary run: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log on successful glossary run: %v", err)
|
||||
}
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
if report.Status != "success" || report.ErrorPhase != "" {
|
||||
t.Fatalf("expected successful report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
|
||||
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3295,7 +3313,7 @@ func TestRunProcessExplicitHomophonesProtectedGlossaryTermRejected(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
|
||||
func TestRunProcessExplicitHomophonesMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
t.Cleanup(func() { processProposalLLMClient = nil })
|
||||
|
||||
@@ -3314,22 +3332,29 @@ func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *test
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected failure")
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
||||
t.Fatalf("expected runner_execution error, got %q", stderr.String())
|
||||
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("expected transcript stdout on success: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
|
||||
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log on failed homophones run: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log on successful homophones run: %v", err)
|
||||
}
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
if report.Status != "success" || report.ErrorPhase != "" {
|
||||
t.Fatalf("expected successful report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
|
||||
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3674,7 +3699,7 @@ func TestRunProcessExplicitSpokenWordProtectedGlossaryTermRejected(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
|
||||
func TestRunProcessExplicitSpokenWordMalformedLLMOutputSucceedsWithWarning(t *testing.T) {
|
||||
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
t.Cleanup(func() { processProposalLLMClient = nil })
|
||||
|
||||
@@ -3693,22 +3718,29 @@ func TestRunProcessExplicitSpokenWordMalformedLLMOutputFailsWithErrorLog(t *test
|
||||
"--work-dir-retention", "always",
|
||||
"--report-json", reportPath,
|
||||
}, &stdout, &stderr)
|
||||
if exitCode == 0 {
|
||||
t.Fatal("expected failure")
|
||||
if exitCode != 0 {
|
||||
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
|
||||
}
|
||||
if stdout.Len() != 0 {
|
||||
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
|
||||
if stderr.Len() != 0 {
|
||||
t.Fatalf("expected empty stderr on success, got %q", stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "runner_execution") {
|
||||
t.Fatalf("expected runner_execution error, got %q", stderr.String())
|
||||
parsed, err := schema.ParseTranscriptJSON(stdout.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("expected transcript stdout on success: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 || parsed.Segments[0].Text != "hello" {
|
||||
t.Fatalf("expected unchanged transcript, got %+v", parsed.Segments)
|
||||
}
|
||||
runDir := onlyRunDir(t, workDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
|
||||
t.Fatalf("expected error.log on failed spoken_word run: %v", err)
|
||||
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatalf("did not expect error.log on successful spoken_word run: %v", err)
|
||||
}
|
||||
report := readProcessReport(t, reportPath)
|
||||
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
|
||||
t.Fatalf("expected failed runner_execution report, got %+v", report)
|
||||
if report.Status != "success" || report.ErrorPhase != "" {
|
||||
t.Fatalf("expected successful report, got %+v", report)
|
||||
}
|
||||
if len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) != 1 {
|
||||
t.Fatalf("expected one module warning, got %+v", report.ModuleResults)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,25 +78,22 @@ func (c *subprocessTestLLMClient) CompleteStructured(ctx context.Context, req co
|
||||
}
|
||||
}
|
||||
case "mid_pipeline_fail":
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
if _, ok := out.(*proposal_generation.StructuredCorrectionSet); ok {
|
||||
c.mu.Lock()
|
||||
c.proposals++
|
||||
proposalCall := c.proposals
|
||||
c.mu.Unlock()
|
||||
|
||||
if proposalCall >= 3 {
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("synthetic mid-pipeline failure")
|
||||
}
|
||||
}
|
||||
switch target := out.(type) {
|
||||
case *proposal_generation.StructuredCorrectionSet:
|
||||
*target = proposal_generation.StructuredCorrectionSet{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "Segment", CorrectedText: "Segment", Confidence: 0.99},
|
||||
},
|
||||
}
|
||||
case *validators.LLMValidationResponse:
|
||||
*target = validators.LLMValidationResponse{Validations: nil}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
type ProcessReport struct {
|
||||
@@ -46,18 +47,19 @@ type ReportMetadata struct {
|
||||
}
|
||||
|
||||
type ModuleReport struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy string `json:"replacement_policy,omitempty"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy string `json:"replacement_policy,omitempty"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
ValidatorDecisions []ValidatorDecisionReport `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedReport `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorDecisionReport struct {
|
||||
|
||||
@@ -47,7 +47,7 @@ func (h testChunkProposalHarness) collectEnrichedProposals(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, proposal := range base {
|
||||
for _, proposal := range base.Proposals {
|
||||
sectionIndex := section.Index
|
||||
out = append(out, proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposal,
|
||||
@@ -85,11 +85,11 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy
|
||||
|
||||
func (m deterministicFakeModule) Validators() []Validator { return nil }
|
||||
|
||||
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
|
||||
_ = ctx
|
||||
|
||||
if req.WorkingTranscript == nil || req.Section == nil {
|
||||
return []proposals.CorrectionProposal{}, nil
|
||||
return ProposalResult{}, nil
|
||||
}
|
||||
|
||||
out := make([]proposals.CorrectionProposal, 0)
|
||||
@@ -110,7 +110,7 @@ func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalReques
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
return ProposalResult{Proposals: out}, nil
|
||||
}
|
||||
|
||||
func TestChunkProposalMetadataAssociation(t *testing.T) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
// StructuredLLMClient provides provider-agnostic structured completion.
|
||||
@@ -28,7 +29,7 @@ type TranscriptModule interface {
|
||||
Key() string
|
||||
ReplacementPolicy() proposals.ReplacementPolicy
|
||||
Validators() []Validator
|
||||
Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error)
|
||||
Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error)
|
||||
}
|
||||
|
||||
// Validator evaluates candidate proposals and returns one decision per proposal index.
|
||||
@@ -103,6 +104,11 @@ type ProposalRequest struct {
|
||||
LLMScheduler LLMScheduler `json:"-"`
|
||||
}
|
||||
|
||||
type ProposalResult struct {
|
||||
Proposals []proposals.CorrectionProposal `json:"proposals,omitempty"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// ValidationRequest is the input to validator execution.
|
||||
type ValidationRequest = validators.Request
|
||||
|
||||
|
||||
@@ -50,11 +50,13 @@ func (f *fakeModule) Validators() []Validator {
|
||||
return []Validator{&fakeValidator{}}
|
||||
}
|
||||
|
||||
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
|
||||
return ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -68,12 +70,12 @@ func TestInterfaceContractsCompileWithFakes(t *testing.T) {
|
||||
t.Fatalf("unexpected module key: %q", got)
|
||||
}
|
||||
|
||||
proposalsOut, err := module.Propose(context.Background(), ProposalRequest{})
|
||||
proposalResult, err := module.Propose(context.Background(), ProposalRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected propose error: %v", err)
|
||||
}
|
||||
if len(proposalsOut) != 1 {
|
||||
t.Fatalf("expected one proposal, got %d", len(proposalsOut))
|
||||
if len(proposalResult.Proposals) != 1 {
|
||||
t.Fatalf("expected one proposal, got %d", len(proposalResult.Proposals))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,10 +19,10 @@ func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
func (m noopModule) Validators() []contracts.Validator { return nil }
|
||||
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
return nil, nil
|
||||
return contracts.ProposalResult{}, nil
|
||||
}
|
||||
|
||||
func TestKnownModuleKeyRecognition(t *testing.T) {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
|
||||
@@ -68,6 +69,7 @@ type Request struct {
|
||||
type Result struct {
|
||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
@@ -156,6 +158,12 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
if isMalformedStructuredOutputError(callErr) {
|
||||
return Result{
|
||||
Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)},
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr)
|
||||
}
|
||||
|
||||
@@ -168,9 +176,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
CorrectedText: raw.CorrectedText,
|
||||
Confidence: raw.Confidence,
|
||||
}
|
||||
if err := candidate.Validate(); err != nil {
|
||||
return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err)
|
||||
}
|
||||
|
||||
corrections = append(corrections, candidate)
|
||||
enrichedCandidate := proposals.EnrichedCorrectionProposal{
|
||||
@@ -191,6 +196,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
return Result{
|
||||
Corrections: corrections,
|
||||
Enriched: enriched,
|
||||
Warnings: nil,
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
@@ -240,6 +246,48 @@ func errPayload(err error) any {
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
|
||||
func isMalformedStructuredOutputError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
} {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning {
|
||||
warning := stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeProposalGeneration,
|
||||
ReasonCode: "proposal_response_malformed",
|
||||
Message: strings.TrimSpace(err.Error()),
|
||||
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
|
||||
}
|
||||
if section != nil {
|
||||
sectionIndex := section.Index
|
||||
warning.SectionIndex = §ionIndex
|
||||
}
|
||||
return warning
|
||||
}
|
||||
|
||||
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
||||
if artifacts.ErrorPayloadPath != "" {
|
||||
return artifacts.ErrorPayloadPath
|
||||
}
|
||||
return artifacts.ResponsePayloadPath
|
||||
}
|
||||
|
||||
type diagnosticsWriterAdapter struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
}
|
||||
|
||||
@@ -224,12 +224,12 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
||||
func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9},
|
||||
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -237,9 +237,38 @@ func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid structured correction") {
|
||||
t.Fatalf("expected structured response validation failure, got %v", err)
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected invalid correction to survive generation, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 1 {
|
||||
t.Fatalf("expected one correction, got %+v", result)
|
||||
}
|
||||
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
|
||||
}
|
||||
if len(result.Warnings) != 0 {
|
||||
t.Fatalf("did not expect warnings for individually invalid corrections, got %+v", result.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("malformed structured output")}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed structured output to downgrade to warning, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
|
||||
t.Fatalf("expected no proposals on malformed response, got %+v", result)
|
||||
}
|
||||
if len(result.Warnings) != 1 {
|
||||
t.Fatalf("expected one warning, got %+v", result.Warnings)
|
||||
}
|
||||
if result.Warnings[0].ReasonCode != "proposal_response_malformed" {
|
||||
t.Fatalf("unexpected warning: %+v", result.Warnings[0])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -147,6 +147,8 @@ func skipReasonMessage(reason ProposalSkipReason) string {
|
||||
return "original_text matched multiple spans under require_unique policy"
|
||||
case SkipReasonNoEffect:
|
||||
return "original_text and corrected_text are identical"
|
||||
case SkipReasonEmptyResultingText:
|
||||
return "proposal would leave the segment empty"
|
||||
case SkipReasonInvalidProposal:
|
||||
return "proposal failed structural or policy validation"
|
||||
default:
|
||||
|
||||
@@ -15,6 +15,7 @@ const (
|
||||
SkipReasonMissingOriginalText ProposalSkipReason = "missing_original_text"
|
||||
SkipReasonAmbiguousOriginal ProposalSkipReason = "ambiguous_original_text"
|
||||
SkipReasonNoEffect ProposalSkipReason = "no_effect"
|
||||
SkipReasonEmptyResultingText ProposalSkipReason = "empty_resulting_segment"
|
||||
SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal"
|
||||
)
|
||||
|
||||
@@ -62,6 +63,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
|
||||
}
|
||||
|
||||
corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
|
||||
if strings.TrimSpace(corrected) == "" {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
|
||||
}
|
||||
return SegmentPreviewResult{
|
||||
Applicable: true,
|
||||
CorrectedSegmentText: corrected,
|
||||
@@ -70,6 +74,9 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
|
||||
|
||||
case ReplacementPolicyReplaceAll:
|
||||
corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
|
||||
if strings.TrimSpace(corrected) == "" {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText}
|
||||
}
|
||||
return SegmentPreviewResult{
|
||||
Applicable: true,
|
||||
CorrectedSegmentText: corrected,
|
||||
|
||||
@@ -121,6 +121,42 @@ func TestPreviewProposalForSegmentNoEffectReplacement(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonEmpty(t *testing.T) {
|
||||
segment := &schema.Segment{ID: 8, Text: "uh hello"}
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh ",
|
||||
CorrectedText: "",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
if !result.Applicable {
|
||||
t.Fatalf("expected applicable preview, got skip reason %q", result.SkipReason)
|
||||
}
|
||||
if result.CorrectedSegmentText != "hello" {
|
||||
t.Fatalf("unexpected corrected text: %q", result.CorrectedSegmentText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentRejectsEmptyResultingSegment(t *testing.T) {
|
||||
segment := &schema.Segment{ID: 8, Text: "uh"}
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh",
|
||||
CorrectedText: "",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
if result.Applicable {
|
||||
t.Fatal("expected non-applicable preview")
|
||||
}
|
||||
if result.SkipReason != SkipReasonEmptyResultingText {
|
||||
t.Fatalf("expected skip reason %q, got %q", SkipReasonEmptyResultingText, result.SkipReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentPreservesInputSegment(t *testing.T) {
|
||||
segment := &schema.Segment{ID: 9, Speaker: "A", Start: 1.0, End: 2.0, Text: "rank rank"}
|
||||
original := *segment
|
||||
|
||||
@@ -38,9 +38,6 @@ func (p CorrectionProposal) Validate() error {
|
||||
if strings.TrimSpace(p.OriginalText) == "" {
|
||||
return fmt.Errorf("proposal original_text must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(p.CorrectedText) == "" {
|
||||
return fmt.Errorf("proposal corrected_text must not be empty")
|
||||
}
|
||||
if p.Confidence < 0.0 || p.Confidence > 1.0 {
|
||||
return fmt.Errorf("proposal confidence must be between 0.0 and 1.0")
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
|
||||
func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 42,
|
||||
OriginalText: "gestures",
|
||||
@@ -43,12 +43,8 @@ func TestCorrectionProposalValidate_InvalidEmptyCorrectedText(t *testing.T) {
|
||||
Confidence: 0.95,
|
||||
}
|
||||
|
||||
err := proposal.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("expected validation error, got nil")
|
||||
}
|
||||
if err.Error() != "proposal corrected_text must not be empty" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
if err := proposal.Validate(); err != nil {
|
||||
t.Fatalf("expected empty corrected_text to be allowed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
|
||||
)
|
||||
|
||||
@@ -37,18 +38,19 @@ type ValidationScheduler = contracts.LLMScheduler
|
||||
|
||||
// ModuleResult captures deterministic per-module execution output.
|
||||
type ModuleResult struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Status string `json:"status"`
|
||||
ProposalCount int `json:"proposal_count"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
|
||||
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
|
||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
type ValidatorDecisionRecord struct {
|
||||
@@ -176,6 +178,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
ReplacementPolicy: policy,
|
||||
Status: ModuleStatusFailed,
|
||||
ProposalCount: pipelineResult.ProposalCount,
|
||||
Warnings: pipelineResult.Warnings,
|
||||
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
||||
ValidatorRejected: pipelineResult.ValidatorRejected,
|
||||
ErrorMessage: pipelineErr.Error(),
|
||||
@@ -199,6 +202,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
ReplacementPolicy: policy,
|
||||
Status: ModuleStatusSuccess,
|
||||
ProposalCount: pipelineResult.ProposalCount,
|
||||
Warnings: pipelineResult.Warnings,
|
||||
ValidatorDecisions: pipelineResult.ValidatorDecisions,
|
||||
ValidatorRejected: pipelineResult.ValidatorRejected,
|
||||
AppliedChanges: applyResult.Applied,
|
||||
@@ -232,12 +236,14 @@ type collectSectionProposalsInput struct {
|
||||
type sectionProposals struct {
|
||||
meta contracts.SectionMetadata
|
||||
corrected []proposals.CorrectionProposal
|
||||
warnings []stagewarnings.StageWarning
|
||||
}
|
||||
|
||||
type sectionProposalResult struct {
|
||||
sectionPos int
|
||||
section chunking.Section
|
||||
corrected []proposals.CorrectionProposal
|
||||
warnings []stagewarnings.StageWarning
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -247,12 +253,14 @@ type sectionValidationResult struct {
|
||||
approved []proposals.EnrichedCorrectionProposal
|
||||
decisions []ValidatorDecisionRecord
|
||||
rejected []ValidatorRejectedChange
|
||||
warnings []stagewarnings.StageWarning
|
||||
err error
|
||||
}
|
||||
|
||||
type modulePipelineResult struct {
|
||||
ProposalCount int
|
||||
Approved []proposals.EnrichedCorrectionProposal
|
||||
Warnings []stagewarnings.StageWarning
|
||||
ValidatorDecisions []ValidatorDecisionRecord
|
||||
ValidatorRejected []ValidatorRejectedChange
|
||||
}
|
||||
@@ -271,7 +279,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
meta := contracts.SectionMetadataFromSection(section)
|
||||
corrected, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
|
||||
proposalResult, err := input.Module.Propose(withModuleInstanceContext(runCtx, input.Spec.InstanceName), contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
Config: input.Config,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
@@ -291,7 +299,8 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
case results <- sectionProposalResult{
|
||||
sectionPos: sectionPos,
|
||||
section: section,
|
||||
corrected: corrected,
|
||||
corrected: proposalResult.Proposals,
|
||||
warnings: proposalResult.Warnings,
|
||||
err: err,
|
||||
}:
|
||||
case <-runCtx.Done():
|
||||
@@ -308,6 +317,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
||||
out := modulePipelineResult{
|
||||
Approved: make([]proposals.EnrichedCorrectionProposal, 0),
|
||||
Warnings: make([]stagewarnings.StageWarning, 0),
|
||||
ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
|
||||
ValidatorRejected: make([]ValidatorRejectedChange, 0),
|
||||
}
|
||||
@@ -352,6 +362,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
if firstErr != nil {
|
||||
continue
|
||||
}
|
||||
out.Warnings = append(out.Warnings, result.warnings...)
|
||||
pending[result.sectionPos] = result
|
||||
|
||||
for {
|
||||
@@ -401,6 +412,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
approved: validated.approved,
|
||||
decisions: validated.decisions,
|
||||
rejected: validated.rejected,
|
||||
warnings: validated.warnings,
|
||||
err: err,
|
||||
}
|
||||
}(nextSectionToProcess, sectionEnriched, sectionMeta)
|
||||
@@ -424,6 +436,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
break
|
||||
}
|
||||
out.Approved = append(out.Approved, res.approved...)
|
||||
out.Warnings = append(out.Warnings, res.warnings...)
|
||||
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
|
||||
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
|
||||
}
|
||||
@@ -440,6 +453,38 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
}
|
||||
return validatorOrder[out.ValidatorRejected[i].ValidatorName] < validatorOrder[out.ValidatorRejected[j].ValidatorName]
|
||||
})
|
||||
sort.SliceStable(out.Warnings, func(i, j int) bool {
|
||||
leftSection, rightSection := -1, -1
|
||||
if out.Warnings[i].SectionIndex != nil {
|
||||
leftSection = *out.Warnings[i].SectionIndex
|
||||
}
|
||||
if out.Warnings[j].SectionIndex != nil {
|
||||
rightSection = *out.Warnings[j].SectionIndex
|
||||
}
|
||||
if leftSection != rightSection {
|
||||
return leftSection < rightSection
|
||||
}
|
||||
leftBatch, rightBatch := -1, -1
|
||||
if out.Warnings[i].BatchIndex != nil {
|
||||
leftBatch = *out.Warnings[i].BatchIndex
|
||||
}
|
||||
if out.Warnings[j].BatchIndex != nil {
|
||||
rightBatch = *out.Warnings[j].BatchIndex
|
||||
}
|
||||
if leftBatch != rightBatch {
|
||||
return leftBatch < rightBatch
|
||||
}
|
||||
if out.Warnings[i].ValidatorName != out.Warnings[j].ValidatorName {
|
||||
return validatorOrder[out.Warnings[i].ValidatorName] < validatorOrder[out.Warnings[j].ValidatorName]
|
||||
}
|
||||
if out.Warnings[i].Scope != out.Warnings[j].Scope {
|
||||
return out.Warnings[i].Scope < out.Warnings[j].Scope
|
||||
}
|
||||
if out.Warnings[i].ReasonCode != out.Warnings[j].ReasonCode {
|
||||
return out.Warnings[i].ReasonCode < out.Warnings[j].ReasonCode
|
||||
}
|
||||
return out.Warnings[i].Message < out.Warnings[j].Message
|
||||
})
|
||||
|
||||
if firstErr != nil {
|
||||
return out, firstErr
|
||||
@@ -467,11 +512,13 @@ type validateSectionCandidatesResult struct {
|
||||
approved []proposals.EnrichedCorrectionProposal
|
||||
decisions []ValidatorDecisionRecord
|
||||
rejected []ValidatorRejectedChange
|
||||
warnings []stagewarnings.StageWarning
|
||||
}
|
||||
|
||||
func validateSectionCandidates(ctx context.Context, input validateSectionCandidatesInput) (validateSectionCandidatesResult, error) {
|
||||
decisions := make([]ValidatorDecisionRecord, 0)
|
||||
rejected := make([]ValidatorRejectedChange, 0)
|
||||
warnings := make([]stagewarnings.StageWarning, 0)
|
||||
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||
|
||||
for _, validator := range input.Validators {
|
||||
@@ -512,6 +559,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
rejected: rejected,
|
||||
warnings: warnings,
|
||||
}, fmt.Errorf("validator %q failed: %w", validator.Name(), err)
|
||||
}
|
||||
if err := validators.EnforceDecisionCardinality(eligible, vResult.Decisions); err != nil {
|
||||
@@ -519,8 +567,10 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
rejected: rejected,
|
||||
warnings: warnings,
|
||||
}, fmt.Errorf("validator %q cardinality failed: %w", validator.Name(), err)
|
||||
}
|
||||
warnings = append(warnings, vResult.Warnings...)
|
||||
|
||||
nextEligible := make([]proposals.EnrichedCorrectionProposal, 0, len(eligible))
|
||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(eligible))
|
||||
@@ -562,6 +612,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
approved: eligible,
|
||||
decisions: decisions,
|
||||
rejected: rejected,
|
||||
warnings: warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -47,11 +47,12 @@ type fakeModule struct {
|
||||
func (m fakeModule) Key() string { return m.key }
|
||||
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if m.proposeF == nil {
|
||||
return nil, nil
|
||||
return contracts.ProposalResult{}, nil
|
||||
}
|
||||
return m.proposeF(req)
|
||||
proposalsOut, err := m.proposeF(req)
|
||||
return contracts.ProposalResult{Proposals: proposalsOut}, err
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
@@ -1191,7 +1192,7 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.T) {
|
||||
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
@@ -1209,15 +1210,21 @@ func TestRunnerLLMValidatorMalformedResponseFailsWithPartialProgress(t *testing.
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected llm validator failure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed validator response to downgrade, got %v", err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
||||
t.Fatalf("expected partial progress retained")
|
||||
}
|
||||
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
|
||||
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
|
||||
}
|
||||
if len(out.ModuleResults[1].Warnings) != 1 || out.ModuleResults[1].Warnings[0].ReasonCode != validators.ReasonValidatorMalformed {
|
||||
t.Fatalf("expected malformed warning, got %+v", out.ModuleResults[1].Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
|
||||
func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
@@ -1232,12 +1239,12 @@ func TestRunnerLLMValidatorMissingDecisionFails(t *testing.T) {
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected missing decision failure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing decision downgrade, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||
@@ -1255,8 +1262,8 @@ func TestRunnerLLMValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
|
||||
ValidationLLMClient: client,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate decision failure")
|
||||
if err != nil {
|
||||
t.Fatalf("expected duplicate decision downgrade, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1355,7 +1362,7 @@ type proposalGenerationModule struct {
|
||||
func (m proposalGenerationModule) Key() string { return m.key }
|
||||
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
@@ -1372,9 +1379,9 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return result.Corrections, nil
|
||||
return contracts.ProposalResult{Proposals: result.Corrections, Warnings: result.Warnings}, nil
|
||||
}
|
||||
|
||||
type fakeProposalStructuredClient struct {
|
||||
|
||||
@@ -4,8 +4,35 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
)
|
||||
|
||||
type ProposalShapeValidator struct{}
|
||||
|
||||
func (v ProposalShapeValidator) Name() string { return "proposal_shape" }
|
||||
|
||||
func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
switch {
|
||||
case c.TargetSegmentID <= 0:
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
|
||||
case strings.TrimSpace(c.OriginalText) == "":
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
|
||||
case c.Confidence < 0.0 || c.Confidence > 1.0:
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
|
||||
default:
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
}
|
||||
}
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
|
||||
}
|
||||
|
||||
type ConfidenceThresholdValidator struct{}
|
||||
|
||||
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
|
||||
@@ -62,10 +89,23 @@ type NonEmptyCorrectionValidator struct{}
|
||||
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" }
|
||||
|
||||
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
segmentsByID := make(map[int]schema.Segment)
|
||||
if req.WorkingTranscript != nil {
|
||||
for _, seg := range req.WorkingTranscript.Segments {
|
||||
segmentsByID[seg.ID] = seg
|
||||
}
|
||||
}
|
||||
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
if strings.TrimSpace(c.CorrectedText) == "" {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty"))
|
||||
segment, ok := segmentsByID[c.TargetSegmentID]
|
||||
if !ok {
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
continue
|
||||
}
|
||||
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy)
|
||||
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
|
||||
continue
|
||||
}
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
@@ -78,7 +79,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
maxTokens = req.Config.ValidationMaxPromptTokens
|
||||
}
|
||||
|
||||
batches, err := ChunkLLMValidationItems(validationReq.Items, maxTokens, v.estimator)
|
||||
warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
|
||||
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
|
||||
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
|
||||
if len(itemsForBatching) == 0 {
|
||||
all := append([]Decision(nil), immediate...)
|
||||
all = append(all, oversized...)
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
@@ -140,12 +154,19 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
if isMalformedStructuredOutputError(err) {
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
||||
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
||||
continue
|
||||
}
|
||||
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
||||
}
|
||||
|
||||
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err)
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
||||
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
||||
continue
|
||||
}
|
||||
for i := range batchDecisions {
|
||||
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
||||
@@ -154,12 +175,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
|
||||
all := append([]Decision(nil), immediate...)
|
||||
all = append(all, oversized...)
|
||||
all = append(all, llmDecisions...)
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
return Result{ValidatorName: v.name, Decisions: all}, nil
|
||||
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
|
||||
}
|
||||
|
||||
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
|
||||
@@ -301,3 +323,98 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
|
||||
}
|
||||
return decisions, nil
|
||||
}
|
||||
|
||||
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
|
||||
out := make([]Decision, 0)
|
||||
for _, item := range items {
|
||||
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
||||
if err != nil || singleTokens <= maxPromptTokens {
|
||||
continue
|
||||
}
|
||||
out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
|
||||
out := make([]stagewarnings.StageWarning, 0)
|
||||
for _, item := range items {
|
||||
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
||||
if err != nil || singleTokens <= maxPromptTokens {
|
||||
continue
|
||||
}
|
||||
out = append(out, stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeValidator,
|
||||
ValidatorName: validatorName,
|
||||
ReasonCode: ReasonValidatorInputTooLarge,
|
||||
Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
|
||||
if len(decisions) == 0 {
|
||||
return append([]LLMValidationItem(nil), items...)
|
||||
}
|
||||
rejected := make(map[int]struct{}, len(decisions))
|
||||
for _, decision := range decisions {
|
||||
rejected[decision.ProposalIndex] = struct{}{}
|
||||
}
|
||||
out := make([]LLMValidationItem, 0, len(items))
|
||||
for _, item := range items {
|
||||
if _, ok := rejected[item.CorrectionIndex]; ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
|
||||
out := make([]Decision, 0, len(items))
|
||||
for _, item := range items {
|
||||
out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
|
||||
idx := batchIndex
|
||||
return stagewarnings.StageWarning{
|
||||
Scope: stagewarnings.ScopeValidator,
|
||||
ValidatorName: validatorName,
|
||||
BatchIndex: &idx,
|
||||
ReasonCode: reasonCode,
|
||||
Message: strings.TrimSpace(message),
|
||||
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
|
||||
}
|
||||
}
|
||||
|
||||
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
||||
if artifacts.ErrorPayloadPath != "" {
|
||||
return artifacts.ErrorPayloadPath
|
||||
}
|
||||
return artifacts.ResponsePayloadPath
|
||||
}
|
||||
|
||||
func isMalformedStructuredOutputError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, marker := range []string{
|
||||
"malformed structured output",
|
||||
"decode structured output:",
|
||||
"decode provider response envelope:",
|
||||
"provider response missing choices",
|
||||
"provider response missing assistant message content",
|
||||
"provider response assistant message content is empty",
|
||||
"provider response assistant message content is not valid JSON",
|
||||
} {
|
||||
if strings.Contains(msg, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -248,29 +248,38 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
||||
t.Fatalf("expected malformed output error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed output downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "response invalid") {
|
||||
t.Fatalf("expected missing decision error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing decision downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||
@@ -278,20 +287,74 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate") {
|
||||
t.Fatalf("expected duplicate decision error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected duplicate decision downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) {
|
||||
func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
_, err := v.Validate(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown") {
|
||||
t.Fatalf("expected unknown index error, got %v", err)
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected unknown index downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
|
||||
huge := strings.Repeat("gestures ", 200)
|
||||
req := Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Text: huge},
|
||||
{ID: 2, Text: "There were gestures at the temple.", Categories: []string{"narration"}},
|
||||
}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
},
|
||||
ModuleKey: "homophones",
|
||||
ModuleInstance: "homophones",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
}
|
||||
req.LLMClient = client
|
||||
cfg := config.Default()
|
||||
cfg.ValidationMaxPromptTokens = 200
|
||||
req.Config = &cfg
|
||||
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected oversize downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 2 {
|
||||
t.Fatalf("expected two decisions, got %+v", res.Decisions)
|
||||
}
|
||||
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved {
|
||||
t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
|
||||
}
|
||||
if !res.Decisions[1].Approved {
|
||||
t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
|
||||
}
|
||||
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge {
|
||||
t.Fatalf("expected one oversize warning, got %+v", res.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,16 +8,22 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
||||
)
|
||||
|
||||
const (
|
||||
ReasonApproved = "approved"
|
||||
ReasonLowConfidence = "low_confidence"
|
||||
ReasonMissingOriginalText = "missing_original_text"
|
||||
ReasonMissingTargetSegment = "missing_target_segment"
|
||||
ReasonEmptyCorrectedText = "empty_corrected_text"
|
||||
ReasonNoEffect = "no_effect"
|
||||
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
||||
ReasonApproved = "approved"
|
||||
ReasonLowConfidence = "low_confidence"
|
||||
ReasonMissingOriginalText = "missing_original_text"
|
||||
ReasonMissingTargetSegment = "missing_target_segment"
|
||||
ReasonEmptyResultingText = "empty_resulting_segment"
|
||||
ReasonNoEffect = "no_effect"
|
||||
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
||||
ReasonInvalidTargetSegment = "invalid_target_segment_id"
|
||||
ReasonEmptyOriginalText = "empty_original_text"
|
||||
ReasonInvalidConfidence = "invalid_confidence"
|
||||
ReasonValidatorMalformed = "validator_response_malformed"
|
||||
ReasonValidatorInputTooLarge = "validator_input_too_large"
|
||||
)
|
||||
|
||||
// Request is the runtime input shared by deterministic validators.
|
||||
@@ -45,8 +51,9 @@ type Decision struct {
|
||||
|
||||
// Result is one validator output containing exactly one decision per proposal index.
|
||||
type Result struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Decisions []Decision `json:"decisions"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Decisions []Decision `json:"decisions"`
|
||||
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// ValidationScheduler provides bounded execution for validator LLM calls.
|
||||
|
||||
@@ -68,6 +68,31 @@ func TestConfidenceThresholdValidator(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalShapeValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "teh", "the", 0.9),
|
||||
mkCandidate(1, 0, "teh", "the", 0.9),
|
||||
mkCandidate(2, 1, " ", "the", 0.9),
|
||||
mkCandidate(3, 1, "teh", "the", 1.5),
|
||||
}}
|
||||
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
}
|
||||
if !res.Decisions[0].Approved {
|
||||
t.Fatalf("expected proposal 0 approved")
|
||||
}
|
||||
if res.Decisions[1].ReasonCode != ReasonInvalidTargetSegment {
|
||||
t.Fatalf("expected invalid target segment rejection, got %+v", res.Decisions[1])
|
||||
}
|
||||
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
|
||||
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
|
||||
}
|
||||
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
|
||||
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginalTextPresenceValidator(t *testing.T) {
|
||||
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
@@ -90,10 +115,13 @@ func TestOriginalTextPresenceValidator(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNonEmptyCorrectionValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "hello", " ", 0.9),
|
||||
}}
|
||||
req := Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "hello world", " ", 0.9),
|
||||
}}
|
||||
res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
@@ -101,8 +129,8 @@ func TestNonEmptyCorrectionValidator(t *testing.T) {
|
||||
if !res.Decisions[0].Approved {
|
||||
t.Fatalf("expected proposal 0 approved")
|
||||
}
|
||||
if res.Decisions[1].ReasonCode != ReasonEmptyCorrectedText {
|
||||
t.Fatalf("expected empty_corrected_text, got %+v", res.Decisions[1])
|
||||
if res.Decisions[1].ReasonCode != ReasonEmptyResultingText {
|
||||
t.Fatalf("expected empty_resulting_segment, got %+v", res.Decisions[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,9 +179,14 @@ func TestStableReasonCodes(t *testing.T) {
|
||||
ReasonLowConfidence,
|
||||
ReasonMissingOriginalText,
|
||||
ReasonMissingTargetSegment,
|
||||
ReasonEmptyCorrectedText,
|
||||
ReasonEmptyResultingText,
|
||||
ReasonNoEffect,
|
||||
ReasonProtectedGlossaryTerm,
|
||||
ReasonInvalidTargetSegment,
|
||||
ReasonEmptyOriginalText,
|
||||
ReasonInvalidConfidence,
|
||||
ReasonValidatorMalformed,
|
||||
ReasonValidatorInputTooLarge,
|
||||
}
|
||||
for _, code := range codes {
|
||||
if strings.TrimSpace(code) == "" {
|
||||
|
||||
18
internal/framework/warnings/warnings.go
Normal file
18
internal/framework/warnings/warnings.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package warnings
|
||||
|
||||
type Scope string
|
||||
|
||||
const (
|
||||
ScopeProposalGeneration Scope = "proposal_generation"
|
||||
ScopeValidator Scope = "validator"
|
||||
)
|
||||
|
||||
type StageWarning struct {
|
||||
Scope Scope `json:"scope"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
SectionIndex *int `json:"section_index,omitempty"`
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
BatchIndex *int `json:"batch_index,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return generated.Corrections, nil
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
|
||||
@@ -129,6 +129,7 @@ func TestGlossaryModuleValidatorChain(t *testing.T) {
|
||||
got = append(got, v.Name())
|
||||
}
|
||||
want := []string{
|
||||
"proposal_shape",
|
||||
"no_effect",
|
||||
"original_text_presence",
|
||||
"confidence_threshold",
|
||||
@@ -185,7 +186,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "glossary:proposal" {
|
||||
t.Fatalf("expected one glossary:proposal call, got %+v", client.calls)
|
||||
}
|
||||
if len(out) != 1 || out[0].CorrectedText != "Jesters" {
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "glossary", "*proposal*response-payload.json"))
|
||||
|
||||
@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return generated.Corrections, nil
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
|
||||
@@ -131,6 +131,7 @@ func TestGrammarModuleValidatorChain(t *testing.T) {
|
||||
got = append(got, v.Name())
|
||||
}
|
||||
want := []string{
|
||||
"proposal_shape",
|
||||
"no_effect",
|
||||
"original_text_presence",
|
||||
"confidence_threshold",
|
||||
@@ -187,7 +188,7 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal" {
|
||||
t.Fatalf("expected one grammar:proposal call, got %+v", client.calls)
|
||||
}
|
||||
if len(out) != 1 || out[0].CorrectedText != "Hello, world" {
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Hello, world" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "grammar", "*proposal*response-payload.json"))
|
||||
|
||||
@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return generated.Corrections, nil
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
|
||||
@@ -145,6 +145,7 @@ func TestHomophonesModuleValidatorChain(t *testing.T) {
|
||||
got = append(got, v.Name())
|
||||
}
|
||||
want := []string{
|
||||
"proposal_shape",
|
||||
"no_effect",
|
||||
"original_text_presence",
|
||||
"confidence_threshold",
|
||||
@@ -201,7 +202,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" {
|
||||
t.Fatalf("expected one homophones:proposal call, got %+v", client.calls)
|
||||
}
|
||||
if len(out) != 1 || out[0].CorrectedText != "Jesters" {
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json"))
|
||||
|
||||
@@ -36,7 +36,7 @@ func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
sectionTranscript := transcriptForSection(req.WorkingTranscript, req.Section)
|
||||
sectionIndex := 0
|
||||
if req.Section != nil {
|
||||
@@ -48,7 +48,7 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
}
|
||||
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
@@ -74,9 +74,12 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return generated.Corrections, nil
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
Warnings: generated.Warnings,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func transcriptForSection(transcript *schema.Transcript, section *contracts.SectionMetadata) *schema.Transcript {
|
||||
|
||||
@@ -132,6 +132,7 @@ func TestSpokenWordModuleValidatorChain(t *testing.T) {
|
||||
got = append(got, v.Name())
|
||||
}
|
||||
want := []string{
|
||||
"proposal_shape",
|
||||
"no_effect",
|
||||
"original_text_presence",
|
||||
"confidence_threshold",
|
||||
@@ -188,7 +189,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "spoken_word:proposal" {
|
||||
t.Fatalf("expected one spoken_word:proposal call, got %+v", client.calls)
|
||||
}
|
||||
if len(out) != 1 || out[0].CorrectedText != "I think" {
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "I think" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "spoken_word", "*proposal*response-payload.json"))
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
var builtInChains = map[string][]string{
|
||||
"glossary": {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
KeyConfidenceThreshold,
|
||||
@@ -18,6 +19,7 @@ var builtInChains = map[string][]string{
|
||||
KeyMeaningReversalReview,
|
||||
},
|
||||
"homophones": {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
KeyConfidenceThreshold,
|
||||
@@ -27,6 +29,7 @@ var builtInChains = map[string][]string{
|
||||
KeyMeaningReversalReview,
|
||||
},
|
||||
"spoken_word": {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
KeyConfidenceThreshold,
|
||||
@@ -36,6 +39,7 @@ var builtInChains = map[string][]string{
|
||||
KeyMeaningReversalReview,
|
||||
},
|
||||
"grammar": {
|
||||
KeyProposalShape,
|
||||
KeyNoEffect,
|
||||
KeyOriginalTextPresence,
|
||||
KeyConfidenceThreshold,
|
||||
|
||||
13
internal/validators/proposal_shape/validator.go
Normal file
13
internal/validators/proposal_shape/validator.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package proposal_shape
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "proposal_shape"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
return validatormetadata.Wrap(frameworkvalidators.ProposalShapeValidator{}, validatormetadata.ExecutionClassDeterministic), nil
|
||||
}
|
||||
@@ -11,11 +11,13 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility"
|
||||
)
|
||||
|
||||
const (
|
||||
KeyProposalShape = "proposal_shape"
|
||||
KeyConfidenceThreshold = "confidence_threshold"
|
||||
KeyOriginalTextPresence = "original_text_presence"
|
||||
KeyNonEmptyCorrectedText = "non_empty_corrected_text"
|
||||
@@ -39,6 +41,7 @@ type Registry struct {
|
||||
|
||||
func NewBuiltInRegistry() *Registry {
|
||||
defs := []BuiltInValidatorDefinition{
|
||||
{Key: KeyProposalShape, Build: proposal_shape.New},
|
||||
{Key: KeyConfidenceThreshold, Build: confidence_threshold.New},
|
||||
{Key: KeyOriginalTextPresence, Build: original_text_presence.New},
|
||||
{Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New},
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/no_effect"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/proposal_shape"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/protected_terms"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/validators/spoken_form_plausibility"
|
||||
)
|
||||
@@ -21,6 +22,7 @@ import (
|
||||
func TestBuiltInRegistryRegistersAllKeys(t *testing.T) {
|
||||
r := NewBuiltInRegistry()
|
||||
for _, key := range []string{
|
||||
KeyProposalShape,
|
||||
KeyConfidenceThreshold,
|
||||
KeyOriginalTextPresence,
|
||||
KeyNonEmptyCorrectedText,
|
||||
@@ -52,6 +54,7 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
|
||||
wantClass validatormetadata.ExecutionClass
|
||||
}
|
||||
cases := []validatorCtor{
|
||||
{name: "proposal_shape", key: KeyProposalShape, build: proposal_shape.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
||||
{name: "confidence_threshold", key: KeyConfidenceThreshold, build: confidence_threshold.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
||||
{name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
||||
{name: "non_empty_corrected_text", key: KeyNonEmptyCorrectedText, build: non_empty_corrected_text.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
||||
|
||||
Reference in New Issue
Block a user