Reconcile partial validator decision responses
This commit is contained in:
@@ -26,14 +26,15 @@ type fakeStructuredLLMClient struct {
|
||||
type captureValidationDiagnosticsWriter struct {
|
||||
err error
|
||||
lastRequestMetadata any
|
||||
lastResponsePayload any
|
||||
}
|
||||
|
||||
func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
_ = stage
|
||||
_ = requestPayload
|
||||
_ = responsePayload
|
||||
_ = errorPayload
|
||||
w.lastRequestMetadata = requestMetadata
|
||||
w.lastResponsePayload = responsePayload
|
||||
return InteractionArtifacts{}, w.err
|
||||
}
|
||||
|
||||
@@ -203,49 +204,82 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected missing decision downgrade, got %v", err)
|
||||
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
approve := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Confidence: .9, Reason: "accepted"}
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
reject := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Confidence: .9, Reason: "editorial concern"}
|
||||
}
|
||||
type expectedDecision struct {
|
||||
index int
|
||||
approved bool
|
||||
reason string
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
entries []LLMValidationDecision
|
||||
want []expectedDecision
|
||||
unavailable []int
|
||||
ignored []int
|
||||
}{
|
||||
{"missing and extra", []LLMValidationDecision{approve(7), reject(18), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
|
||||
{"conflicting duplicate", []LLMValidationDecision{approve(7), approve(11), reject(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
||||
{"identical duplicate", []LLMValidationDecision{approve(7), approve(11), approve(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
||||
{"extra only", []LLMValidationDecision{approve(99), reject(18), approve(7), approve(99), approve(11)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
|
||||
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
|
||||
{"reordered", []LLMValidationDecision{reject(18), approve(11), approve(7)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
|
||||
{"invalid confidence", []LLMValidationDecision{approve(7), {CorrectionIndex: 11, Approved: true, Confidence: 1.2}, reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
|
||||
writer := &captureValidationDiagnosticsWriter{}
|
||||
v, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "actors"), mk(18, "gestures", "figures")})
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = writer
|
||||
result, err := v.Validate(context.Background(), req)
|
||||
if err != nil || len(client.calls) != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
|
||||
t.Fatalf("result=%+v calls=%d err=%v", result, len(client.calls), err)
|
||||
}
|
||||
for i, want := range tc.want {
|
||||
got := result.Decisions[i]
|
||||
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) {
|
||||
t.Fatalf("decision[%d]=%+v want=%+v", i, got, want)
|
||||
}
|
||||
if got.ReasonCode == "llm_rejected" && got.Message != "editorial concern" {
|
||||
t.Fatalf("negative reason lost: %+v", got)
|
||||
}
|
||||
}
|
||||
op := result.Operations[0]
|
||||
if op.Succeeded != (len(tc.unavailable) == 0) || !equalIndices(op.ProposalIndices, tc.unavailable) || (op.FailureReason == outcomes.ReasonValidationDecisionsInvalid) != (len(tc.unavailable) > 0) || len(op.Filtered) != 0 {
|
||||
t.Fatalf("operation=%+v", op)
|
||||
}
|
||||
if len(tc.unavailable) > 0 && len(op.RecoveryReasons) != 0 || len(tc.unavailable) == 0 && (len(op.RecoveryReasons) == 1) != (len(tc.ignored) > 0) {
|
||||
t.Fatalf("recovery=%+v", op)
|
||||
}
|
||||
if len(tc.ignored) > 0 && len(tc.unavailable) == 0 && op.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered {
|
||||
t.Fatalf("recovery reason=%+v", op)
|
||||
}
|
||||
payload := writer.lastResponsePayload.(map[string]any)
|
||||
reconciliation := payload["reconciliation"].(validationReconciliation)
|
||||
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
|
||||
t.Fatalf("diagnostic reconciliation=%+v", reconciliation)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||
}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected duplicate decision downgrade, got %v", err)
|
||||
func equalIndices(a, b []int) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
res, err := v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected unknown index downgrade, got %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user