Make module-stage LLM handling resilient and report warnings

This commit is contained in:
2026-05-23 10:07:06 -05:00
parent a84941d681
commit a3655f5540
43 changed files with 856 additions and 217 deletions

View File

@@ -4,8 +4,35 @@ import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
)
type ProposalShapeValidator struct{}
func (v ProposalShapeValidator) Name() string { return "proposal_shape" }
func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result, error) {
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
switch {
case c.TargetSegmentID <= 0:
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
case strings.TrimSpace(c.OriginalText) == "":
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
case c.Confidence < 0.0 || c.Confidence > 1.0:
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
default:
decisions = append(decisions, approval(c.ProposalIndex))
}
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type ConfidenceThresholdValidator struct{}
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
@@ -62,10 +89,23 @@ type NonEmptyCorrectionValidator struct{}
func (v NonEmptyCorrectionValidator) Name() string { return "non_empty_corrected_text" }
func (v NonEmptyCorrectionValidator) Validate(_ context.Context, req Request) (Result, error) {
segmentsByID := make(map[int]schema.Segment)
if req.WorkingTranscript != nil {
for _, seg := range req.WorkingTranscript.Segments {
segmentsByID[seg.ID] = seg
}
}
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
if strings.TrimSpace(c.CorrectedText) == "" {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyCorrectedText, "corrected_text must not be empty"))
segment, ok := segmentsByID[c.TargetSegmentID]
if !ok {
decisions = append(decisions, approval(c.ProposalIndex))
continue
}
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy)
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))

View File

@@ -11,6 +11,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
@@ -78,7 +79,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
maxTokens = req.Config.ValidationMaxPromptTokens
}
batches, err := ChunkLLMValidationItems(validationReq.Items, maxTokens, v.estimator)
warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
if len(itemsForBatching) == 0 {
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
}
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
if err != nil {
return Result{}, err
}
@@ -140,12 +154,19 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
)
}
if err != nil {
if isMalformedStructuredOutputError(err) {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
continue
}
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
}
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
if err != nil {
return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err)
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
continue
}
for i := range batchDecisions {
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
@@ -154,12 +175,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
all = append(all, llmDecisions...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all}, nil
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
}
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
@@ -301,3 +323,98 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
}
return decisions, nil
}
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
out := make([]Decision, 0)
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
}
return out
}
func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
out := make([]stagewarnings.StageWarning, 0)
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
ReasonCode: ReasonValidatorInputTooLarge,
Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
})
}
return out
}
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
if len(decisions) == 0 {
return append([]LLMValidationItem(nil), items...)
}
rejected := make(map[int]struct{}, len(decisions))
for _, decision := range decisions {
rejected[decision.ProposalIndex] = struct{}{}
}
out := make([]LLMValidationItem, 0, len(items))
for _, item := range items {
if _, ok := rejected[item.CorrectionIndex]; ok {
continue
}
out = append(out, item)
}
return out
}
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
out := make([]Decision, 0, len(items))
for _, item := range items {
out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
}
return out
}
func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
idx := batchIndex
return stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
BatchIndex: &idx,
ReasonCode: reasonCode,
Message: strings.TrimSpace(message),
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
}
}
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
if artifacts.ErrorPayloadPath != "" {
return artifacts.ErrorPayloadPath
}
return artifacts.ResponsePayloadPath
}
func isMalformedStructuredOutputError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, marker := range []string{
"malformed structured output",
"decode structured output:",
"decode provider response envelope:",
"provider response missing choices",
"provider response missing assistant message content",
"provider response assistant message content is empty",
"provider response assistant message content is not valid JSON",
} {
if strings.Contains(msg, marker) {
return true
}
}
return false
}

View File

@@ -248,29 +248,38 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
}
}
func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "completion failed") {
t.Fatalf("expected malformed output error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed output downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
}
}
func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) {
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "response invalid") {
t.Fatalf("expected missing decision error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
@@ -278,20 +287,74 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "duplicate") {
t.Fatalf("expected duplicate decision error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) {
func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "unknown") {
t.Fatalf("expected unknown index error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected unknown index downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
huge := strings.Repeat("gestures ", 200)
req := Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Text: huge},
{ID: 2, Text: "There were gestures at the temple.", Categories: []string{"narration"}},
}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
},
ModuleKey: "homophones",
ModuleInstance: "homophones",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
}
req.LLMClient = client
cfg := config.Default()
cfg.ValidationMaxPromptTokens = 200
req.Config = &cfg
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected oversize downgrade, got %v", err)
}
if len(res.Decisions) != 2 {
t.Fatalf("expected two decisions, got %+v", res.Decisions)
}
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved {
t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
}
if !res.Decisions[1].Approved {
t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("expected one oversize warning, got %+v", res.Warnings)
}
}

View File

@@ -8,16 +8,22 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
)
const (
ReasonApproved = "approved"
ReasonLowConfidence = "low_confidence"
ReasonMissingOriginalText = "missing_original_text"
ReasonMissingTargetSegment = "missing_target_segment"
ReasonEmptyCorrectedText = "empty_corrected_text"
ReasonNoEffect = "no_effect"
ReasonProtectedGlossaryTerm = "protected_glossary_term"
ReasonApproved = "approved"
ReasonLowConfidence = "low_confidence"
ReasonMissingOriginalText = "missing_original_text"
ReasonMissingTargetSegment = "missing_target_segment"
ReasonEmptyResultingText = "empty_resulting_segment"
ReasonNoEffect = "no_effect"
ReasonProtectedGlossaryTerm = "protected_glossary_term"
ReasonInvalidTargetSegment = "invalid_target_segment_id"
ReasonEmptyOriginalText = "empty_original_text"
ReasonInvalidConfidence = "invalid_confidence"
ReasonValidatorMalformed = "validator_response_malformed"
ReasonValidatorInputTooLarge = "validator_input_too_large"
)
// Request is the runtime input shared by deterministic validators.
@@ -45,8 +51,9 @@ type Decision struct {
// Result is one validator output containing exactly one decision per proposal index.
type Result struct {
ValidatorName string `json:"validator_name"`
Decisions []Decision `json:"decisions"`
ValidatorName string `json:"validator_name"`
Decisions []Decision `json:"decisions"`
Warnings []stagewarnings.StageWarning `json:"warnings,omitempty"`
}
// ValidationScheduler provides bounded execution for validator LLM calls.

View File

@@ -68,6 +68,31 @@ func TestConfidenceThresholdValidator(t *testing.T) {
}
}
func TestProposalShapeValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 0, "teh", "the", 0.9),
mkCandidate(2, 1, " ", "the", 0.9),
mkCandidate(3, 1, "teh", "the", 1.5),
}}
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved {
t.Fatalf("expected proposal 0 approved")
}
if res.Decisions[1].ReasonCode != ReasonInvalidTargetSegment {
t.Fatalf("expected invalid target segment rejection, got %+v", res.Decisions[1])
}
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
}
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
}
}
func TestOriginalTextPresenceValidator(t *testing.T) {
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
@@ -90,10 +115,13 @@ func TestOriginalTextPresenceValidator(t *testing.T) {
}
func TestNonEmptyCorrectionValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "hello", " ", 0.9),
}}
req := Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "hello world", " ", 0.9),
}}
res, err := (NonEmptyCorrectionValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
@@ -101,8 +129,8 @@ func TestNonEmptyCorrectionValidator(t *testing.T) {
if !res.Decisions[0].Approved {
t.Fatalf("expected proposal 0 approved")
}
if res.Decisions[1].ReasonCode != ReasonEmptyCorrectedText {
t.Fatalf("expected empty_corrected_text, got %+v", res.Decisions[1])
if res.Decisions[1].ReasonCode != ReasonEmptyResultingText {
t.Fatalf("expected empty_resulting_segment, got %+v", res.Decisions[1])
}
}
@@ -151,9 +179,14 @@ func TestStableReasonCodes(t *testing.T) {
ReasonLowConfidence,
ReasonMissingOriginalText,
ReasonMissingTargetSegment,
ReasonEmptyCorrectedText,
ReasonEmptyResultingText,
ReasonNoEffect,
ReasonProtectedGlossaryTerm,
ReasonInvalidTargetSegment,
ReasonEmptyOriginalText,
ReasonInvalidConfidence,
ReasonValidatorMalformed,
ReasonValidatorInputTooLarge,
}
for _, code := range codes {
if strings.TrimSpace(code) == "" {