eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Hardened the proposal modules to skip malformed proposals rather than hard failing the entire run
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
Details

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-17 07:19:24 -05:00
parent a84941d681
commit 9c2d8338d7
17 changed files with 436 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
internal/framework/validators/llm_validators_test.go
View File

@@ -259,18 +259,24 @@ func TestLLMBackedValidatorMalformedOutputFails(t *testing.T) {
}
}
func TestLLMBackedValidatorMissingDecisionFails(t *testing.T) {
func TestLLMBackedValidatorMissingDecisionSoftRejects(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "response invalid") {
t.Fatalf("expected missing decision error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved {
t.Fatalf("expected one soft rejection, got %+v", res.Decisions)
}
if res.Decisions[0].ReasonCode != ReasonValidatorMissing {
t.Fatalf("expected %q, got %+v", ReasonValidatorMissing, res.Decisions[0])
}
}
func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
func TestLLMBackedValidatorDuplicateDecisionSoftRejects(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
@@ -278,20 +284,32 @@ func TestLLMBackedValidatorDuplicateDecisionFails(t *testing.T) {
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "duplicate") {
t.Fatalf("expected duplicate decision error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved {
t.Fatalf("expected one soft rejection, got %+v", res.Decisions)
}
if res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("expected %q, got %+v", ReasonValidatorMalformed, res.Decisions[0])
}
}
func TestLLMBackedValidatorUnknownProposalIndexFails(t *testing.T) {
func TestLLMBackedValidatorUnknownProposalIndexSoftRejectsMissing(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
_, err := v.Validate(context.Background(), req)
if err == nil || !strings.Contains(err.Error(), "unknown") {
t.Fatalf("expected unknown index error, got %v", err)
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved {
t.Fatalf("expected one soft rejection, got %+v", res.Decisions)
}
if res.Decisions[0].ReasonCode != ReasonValidatorMissing {
t.Fatalf("expected %q, got %+v", ReasonValidatorMissing, res.Decisions[0])
}
}