eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Preserve validation decisions across failed batches

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:14:33 +00:00
parent 42bde59827
commit 0a19f163c6
7 changed files with 233 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/framework/validators/llm_validators_test.go
View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
@@ -136,7 +137,7 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
}
func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}}
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
@@ -144,11 +145,11 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
if err != nil {
t.Fatalf("expected malformed output downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable || res.Decisions[0].ReasonCode != string(outcomes.ReasonInvalidOutputExhausted) {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
if len(res.Operations) != 1 || res.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
t.Fatalf("expected failed operation, got %+v", res.Operations)
}
}
@@ -158,7 +159,7 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
result, err := v.Validate(context.Background(), req)
if err == nil || len(result.Warnings) != 0 {
if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonInternalFailure {
t.Fatalf("protocol error was downgraded: %+v, %v", result, err)
}
}
@@ -172,7 +173,7 @@ func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
@@ -189,7 +190,7 @@ func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
@@ -203,7 +204,7 @@ func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
if err != nil {
t.Fatalf("expected unknown index downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
@@ -245,14 +246,14 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
if len(res.Decisions) != 2 {
t.Fatalf("expected two decisions, got %+v", res.Decisions)
}
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved {
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable {
t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
}
if !res.Decisions[1].Approved {
t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("expected one oversize warning, got %+v", res.Warnings)
if len(res.Operations) != 2 || res.Operations[0].Reference.Kind != outcomes.OperationValidationInput || res.Operations[0].FailureReason != outcomes.ReasonValidationInputTooLarge || res.Operations[0].Reference.BatchIndex != nil || len(res.Operations[0].ProposalIndices) != 1 || res.Operations[0].ProposalIndices[0] != 0 {
t.Fatalf("expected consolidated oversized input outcome, got %+v", res.Operations)
}
}