package validators import ( "context" "encoding/json" "errors" "strings" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ) type perItemEstimator struct{} func (perItemEstimator) EstimateTokens(payload string) int { return 100 * strings.Count(payload, `"correction_index"`) } type validationReply struct { response LLMValidationResponse completion StructuredCompletionResponse err error } type scriptedValidationClient struct { replies []validationReply calls []StructuredCompletionRequest } func (c *scriptedValidationClient) CompleteStructured(_ context.Context, req StructuredCompletionRequest, target any) (StructuredCompletionResponse, error) { c.calls = append(c.calls, req) if len(c.replies) == 0 { return StructuredCompletionResponse{}, errors.New("unexpected validation call") } reply := c.replies[0] c.replies = c.replies[1:] if reply.err == nil { *target.(*LLMValidationResponse) = reply.response } return reply.completion, reply.err } func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { candidates := []proposals.EnrichedCorrectionProposal{ mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"), } failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"} client := &scriptedValidationClient{replies: []validationReply{ {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, {completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, }} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) req := makeReq(candidates) req.Config.Validation.MaxPromptTokens = 150 req.LLMClient = client result, err := validator.Validate(context.Background(), req) if err != nil { t.Fatal(err) } if len(result.Decisions) != 3 || !result.Decisions[0].Approved || result.Decisions[1].Disposition != DispositionUnavailable || !result.Decisions[2].Approved { t.Fatalf("batch decisions: %+v", result.Decisions) } if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 { t.Fatalf("batch operations: %+v", result.Operations) } if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil { t.Fatal(err) } survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]} nextClient := &scriptedValidationClient{replies: []validationReply{ {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, }} nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal) nextValidator.SetTokenEstimator(perItemEstimator{}) nextReq := makeReq(survivors) nextReq.Config.Validation.MaxPromptTokens = 150 nextReq.LLMClient = nextClient next, err := nextValidator.Validate(context.Background(), nextReq) if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 { t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err) } for _, call := range append(client.calls, nextClient.calls...) { var items []LLMValidationItem if err := json.Unmarshal([]byte(call.Variables["PayloadJSON"]), &items); err != nil { t.Fatal(err) } if len(items) != 1 || items[0].CorrectionIndex != 1 { t.Fatalf("batch references must restart: %+v", items) } } if !next.Decisions[0].Approved || !next.Decisions[1].Approved || next.Decisions[0].ProposalIndex != 0 || next.Decisions[1].ProposalIndex != 2 { t.Fatalf("rebatching lost identities: %+v", next.Decisions) } } func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) { client := &scriptedValidationClient{replies: []validationReply{{ response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}}, completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}}, }}} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client result, err := validator.Validate(context.Background(), req) if err != nil || len(result.Decisions) != 1 || result.Decisions[0].ReasonCode != ReasonMissingValidationDecision || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 || !equalIndices(result.Operations[0].ProposalIndices, []int{0}) { t.Fatalf("domain-invalid repaired response: %+v, %v", result, err) } } func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) { client := &scriptedValidationClient{replies: []validationReply{{ response: LLMValidationResponse{Validations: []LLMValidationDecision{ {CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}, {CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"}, }}, completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}}, }}} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client result, err := validator.Validate(context.Background(), req) if err != nil || len(result.Decisions) != 1 || !result.Decisions[0].Approved || len(result.Operations) != 1 { t.Fatalf("validation result: %+v, %v", result, err) } op := result.Operations[0] if !op.Succeeded || op.Counts.ProviderAttempts != 3 || op.Counts.TransientRetries != 1 || op.Counts.StructuralRepairs != 1 || len(op.RecoveryReasons) != 3 || op.RecoveryReasons[0] != outcomes.RecoveryOutputRepaired || op.RecoveryReasons[1] != outcomes.RecoveryServiceRecovered || op.RecoveryReasons[2] != outcomes.RecoveryValidationDecisionsFiltered || len(op.Filtered) != 0 { t.Fatalf("operation: %+v", op) } } func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) { cause := errors.New("broken embedded schema") client := &scriptedValidationClient{replies: []validationReply{ {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, {err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, }} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures")}) req.Config.Validation.MaxPromptTokens = 150 req.LLMClient = client result, err := validator.Validate(context.Background(), req) if !errors.Is(err, cause) || len(result.Decisions) != 1 || len(result.Operations) != 2 || len(client.calls) != 2 { t.Fatalf("fatal partial result: %+v, calls=%d, err=%v", result, len(client.calls), err) } }