154 lines
8.3 KiB
Go
154 lines
8.3 KiB
Go
package validators
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
|
)
|
|
|
|
type perItemEstimator struct{}
|
|
|
|
func (perItemEstimator) EstimateTokens(payload string) int {
|
|
return 100 * strings.Count(payload, `"correction_index"`)
|
|
}
|
|
|
|
type validationReply struct {
|
|
response LLMValidationResponse
|
|
completion StructuredCompletionResponse
|
|
err error
|
|
}
|
|
|
|
type scriptedValidationClient struct {
|
|
replies []validationReply
|
|
calls []StructuredCompletionRequest
|
|
}
|
|
|
|
func (c *scriptedValidationClient) CompleteStructured(_ context.Context, req StructuredCompletionRequest, target any) (StructuredCompletionResponse, error) {
|
|
c.calls = append(c.calls, req)
|
|
if len(c.replies) == 0 {
|
|
return StructuredCompletionResponse{}, errors.New("unexpected validation call")
|
|
}
|
|
reply := c.replies[0]
|
|
c.replies = c.replies[1:]
|
|
if reply.err == nil {
|
|
*target.(*LLMValidationResponse) = reply.response
|
|
}
|
|
return reply.completion, reply.err
|
|
}
|
|
|
|
func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
|
candidates := []proposals.EnrichedCorrectionProposal{
|
|
mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"),
|
|
}
|
|
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
|
|
client := &scriptedValidationClient{replies: []validationReply{
|
|
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
|
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
|
|
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
|
}}
|
|
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
|
validator.SetTokenEstimator(perItemEstimator{})
|
|
req := makeReq(candidates)
|
|
req.Config.Validation.MaxPromptTokens = 150
|
|
req.LLMClient = client
|
|
result, err := validator.Validate(context.Background(), req)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(result.Decisions) != 3 || !result.Decisions[0].Approved || result.Decisions[1].Disposition != DispositionUnavailable || !result.Decisions[2].Approved {
|
|
t.Fatalf("batch decisions: %+v", result.Decisions)
|
|
}
|
|
if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 {
|
|
t.Fatalf("batch operations: %+v", result.Operations)
|
|
}
|
|
if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
|
|
nextClient := &scriptedValidationClient{replies: []validationReply{
|
|
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
|
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
|
}}
|
|
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
|
|
nextValidator.SetTokenEstimator(perItemEstimator{})
|
|
nextReq := makeReq(survivors)
|
|
nextReq.Config.Validation.MaxPromptTokens = 150
|
|
nextReq.LLMClient = nextClient
|
|
next, err := nextValidator.Validate(context.Background(), nextReq)
|
|
if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 {
|
|
t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err)
|
|
}
|
|
for _, call := range append(client.calls, nextClient.calls...) {
|
|
var items []LLMValidationItem
|
|
if err := json.Unmarshal([]byte(call.Variables["PayloadJSON"]), &items); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(items) != 1 || items[0].CorrectionIndex != 1 {
|
|
t.Fatalf("batch references must restart: %+v", items)
|
|
}
|
|
}
|
|
if !next.Decisions[0].Approved || !next.Decisions[1].Approved || next.Decisions[0].ProposalIndex != 0 || next.Decisions[1].ProposalIndex != 2 {
|
|
t.Fatalf("rebatching lost identities: %+v", next.Decisions)
|
|
}
|
|
}
|
|
|
|
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
|
client := &scriptedValidationClient{replies: []validationReply{{
|
|
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}},
|
|
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
|
|
}}}
|
|
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
result, err := validator.Validate(context.Background(), req)
|
|
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].ReasonCode != ReasonMissingValidationDecision || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 || !equalIndices(result.Operations[0].ProposalIndices, []int{0}) {
|
|
t.Fatalf("domain-invalid repaired response: %+v, %v", result, err)
|
|
}
|
|
}
|
|
|
|
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
|
client := &scriptedValidationClient{replies: []validationReply{{
|
|
response: LLMValidationResponse{Validations: []LLMValidationDecision{
|
|
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
|
|
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
|
|
}},
|
|
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
|
|
}}}
|
|
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
|
req.LLMClient = client
|
|
result, err := validator.Validate(context.Background(), req)
|
|
if err != nil || len(result.Decisions) != 1 || !result.Decisions[0].Approved || len(result.Operations) != 1 {
|
|
t.Fatalf("validation result: %+v, %v", result, err)
|
|
}
|
|
op := result.Operations[0]
|
|
if !op.Succeeded || op.Counts.ProviderAttempts != 3 || op.Counts.TransientRetries != 1 || op.Counts.StructuralRepairs != 1 || len(op.RecoveryReasons) != 3 || op.RecoveryReasons[0] != outcomes.RecoveryOutputRepaired || op.RecoveryReasons[1] != outcomes.RecoveryServiceRecovered || op.RecoveryReasons[2] != outcomes.RecoveryValidationDecisionsFiltered || len(op.Filtered) != 0 {
|
|
t.Fatalf("operation: %+v", op)
|
|
}
|
|
}
|
|
|
|
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
|
|
cause := errors.New("broken embedded schema")
|
|
client := &scriptedValidationClient{replies: []validationReply{
|
|
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
|
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
|
|
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
|
}}
|
|
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
|
validator.SetTokenEstimator(perItemEstimator{})
|
|
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures")})
|
|
req.Config.Validation.MaxPromptTokens = 150
|
|
req.LLMClient = client
|
|
result, err := validator.Validate(context.Background(), req)
|
|
if !errors.Is(err, cause) || len(result.Decisions) != 1 || len(result.Operations) != 2 || len(client.calls) != 2 {
|
|
t.Fatalf("fatal partial result: %+v, calls=%d, err=%v", result, len(client.calls), err)
|
|
}
|
|
}
|