eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Reconcile partial validator decision responses

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 11:25:22 +00:00
parent 7199cb565f
commit d726ea8f6d
7 changed files with 221 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/framework/validators/llm_partial_test.go
View File

@@ -95,11 +95,32 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 {
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].ReasonCode != ReasonMissingValidationDecision || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 || !equalIndices(result.Operations[0].ProposalIndices, []int{0}) {
t.Fatalf("domain-invalid repaired response: %+v, %v", result, err)
}
}
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
}}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if err != nil || len(result.Decisions) != 1 || !result.Decisions[0].Approved || len(result.Operations) != 1 {
t.Fatalf("validation result: %+v, %v", result, err)
}
op := result.Operations[0]
if !op.Succeeded || op.Counts.ProviderAttempts != 3 || op.Counts.TransientRetries != 1 || op.Counts.StructuralRepairs != 1 || len(op.RecoveryReasons) != 3 || op.RecoveryReasons[0] != outcomes.RecoveryOutputRepaired || op.RecoveryReasons[1] != outcomes.RecoveryServiceRecovered || op.RecoveryReasons[2] != outcomes.RecoveryValidationDecisionsFiltered || len(op.Filtered) != 0 {
t.Fatalf("operation: %+v", op)
}
}
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{