package validators import ( "context" "errors" "strings" "testing" "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" ) type perItemEstimator struct{} func (perItemEstimator) EstimateTokens(payload string) int { return 100 * strings.Count(payload, `"correction_index"`) } type validationReply struct { response LLMValidationResponse completion StructuredCompletionResponse err error } type scriptedValidationClient struct { replies []validationReply calls []StructuredCompletionRequest } func (c *scriptedValidationClient) CompleteStructured(_ context.Context, req StructuredCompletionRequest, target any) (StructuredCompletionResponse, error) { c.calls = append(c.calls, req) if len(c.replies) == 0 { return StructuredCompletionResponse{}, errors.New("unexpected validation call") } reply := c.replies[0] c.replies = c.replies[1:] if reply.err == nil { *target.(*LLMValidationResponse) = reply.response } return reply.completion, reply.err } func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { candidates := []proposals.EnrichedCorrectionProposal{ mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"), } failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"} client := &scriptedValidationClient{replies: []validationReply{ {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, {completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, }} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) req := makeReq(candidates) req.Config.Validation.MaxPromptTokens = 150 req.LLMClient = client result, err := validator.Validate(context.Background(), req) if err != nil { t.Fatal(err) } if len(result.Decisions) != 3 || !result.Decisions[0].Approved || result.Decisions[1].Disposition != DispositionUnavailable || !result.Decisions[2].Approved { t.Fatalf("batch decisions: %+v", result.Decisions) } if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 { t.Fatalf("batch operations: %+v", result.Operations) } if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil { t.Fatal(err) } survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]} nextClient := &scriptedValidationClient{replies: []validationReply{ {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, }} nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal) nextValidator.SetTokenEstimator(perItemEstimator{}) nextReq := makeReq(survivors) nextReq.Config.Validation.MaxPromptTokens = 150 nextReq.LLMClient = nextClient next, err := nextValidator.Validate(context.Background(), nextReq) if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 { t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err) } } func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) { client := &scriptedValidationClient{replies: []validationReply{{ response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}}, completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}}, }}} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client result, err := validator.Validate(context.Background(), req) if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 { t.Fatalf("domain-invalid repaired response: %+v, %v", result, err) } } func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) { cause := errors.New("broken embedded schema") client := &scriptedValidationClient{replies: []validationReply{ {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, {err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, }} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures")}) req.Config.Validation.MaxPromptTokens = 150 req.LLMClient = client result, err := validator.Validate(context.Background(), req) if !errors.Is(err, cause) || len(result.Decisions) != 1 || len(result.Operations) != 2 || len(client.calls) != 2 { t.Fatalf("fatal partial result: %+v, calls=%d, err=%v", result, len(client.calls), err) } }