Preserve validation decisions across failed batches

2026-09-24 17:14:33 +00:00
parent 42bde59827
commit 0a19f163c6
7 changed files with 233 additions and 73 deletions
--- a/internal/framework/validators/llm_partial_test.go
+++ b/internal/framework/validators/llm_partial_test.go
@@ -0,0 +1,119 @@
+package validators
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
+	"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
+)
+
+type perItemEstimator struct{}
+
+func (perItemEstimator) EstimateTokens(payload string) int {
+	return 100 * strings.Count(payload, `"correction_index"`)
+}
+
+type validationReply struct {
+	response   LLMValidationResponse
+	completion StructuredCompletionResponse
+	err        error
+}
+
+type scriptedValidationClient struct {
+	replies []validationReply
+	calls   []StructuredCompletionRequest
+}
+
+func (c *scriptedValidationClient) CompleteStructured(_ context.Context, req StructuredCompletionRequest, target any) (StructuredCompletionResponse, error) {
+	c.calls = append(c.calls, req)
+	if len(c.replies) == 0 {
+		return StructuredCompletionResponse{}, errors.New("unexpected validation call")
+	}
+	reply := c.replies[0]
+	c.replies = c.replies[1:]
+	if reply.err == nil {
+		*target.(*LLMValidationResponse) = reply.response
+	}
+	return reply.completion, reply.err
+}
+
+func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
+	candidates := []proposals.EnrichedCorrectionProposal{
+		mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"),
+	}
+	failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
+	client := &scriptedValidationClient{replies: []validationReply{
+		{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
+		{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
+		{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
+	}}
+	validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
+	validator.SetTokenEstimator(perItemEstimator{})
+	req := makeReq(candidates)
+	req.Config.Validation.MaxPromptTokens = 150
+	req.LLMClient = client
+	result, err := validator.Validate(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result.Decisions) != 3 || !result.Decisions[0].Approved || result.Decisions[1].Disposition != DispositionUnavailable || !result.Decisions[2].Approved {
+		t.Fatalf("batch decisions: %+v", result.Decisions)
+	}
+	if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 {
+		t.Fatalf("batch operations: %+v", result.Operations)
+	}
+	if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil {
+		t.Fatal(err)
+	}
+
+	survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
+	nextClient := &scriptedValidationClient{replies: []validationReply{
+		{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
+		{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
+	}}
+	nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
+	nextValidator.SetTokenEstimator(perItemEstimator{})
+	nextReq := makeReq(survivors)
+	nextReq.Config.Validation.MaxPromptTokens = 150
+	nextReq.LLMClient = nextClient
+	next, err := nextValidator.Validate(context.Background(), nextReq)
+	if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 {
+		t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err)
+	}
+}
+
+func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
+	client := &scriptedValidationClient{replies: []validationReply{{
+		response:   LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}},
+		completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
+	}}}
+	validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
+	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
+	req.LLMClient = client
+	result, err := validator.Validate(context.Background(), req)
+	if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 {
+		t.Fatalf("domain-invalid repaired response: %+v, %v", result, err)
+	}
+}
+
+func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
+	cause := errors.New("broken embedded schema")
+	client := &scriptedValidationClient{replies: []validationReply{
+		{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
+		{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
+		{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
+	}}
+	validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
+	validator.SetTokenEstimator(perItemEstimator{})
+	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures")})
+	req.Config.Validation.MaxPromptTokens = 150
+	req.LLMClient = client
+	result, err := validator.Validate(context.Background(), req)
+	if !errors.Is(err, cause) || len(result.Decisions) != 1 || len(result.Operations) != 2 || len(client.calls) != 2 {
+		t.Fatalf("fatal partial result: %+v, calls=%d, err=%v", result, len(client.calls), err)
+	}
+}