From d726ea8f6d191718708a59a96b0ca0291cb1beed Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 25 Sep 2026 11:25:22 +0000 Subject: [PATCH] Reconcile partial validator decision responses --- docs/roadmap/implementation.md | 5 + internal/framework/outcomes/outcomes.go | 7 +- internal/framework/runner/runner_test.go | 43 +++++++ .../framework/validators/llm_partial_test.go | 23 +++- .../framework/validators/llm_validators.go | 104 ++++++++++------- .../validators/llm_validators_test.go | 110 ++++++++++++------ internal/framework/validators/models.go | 27 +++-- 7 files changed, 221 insertions(+), 98 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 56a5470..958bdfe 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -143,6 +143,11 @@ cannot accept them as authoritative verdicts. ## Stage 1 — Reconcile Model Decisions and Preserve Valid Work +**Completed:** 2026-09-25. Reconciliation preserves usable decisions, attributes +unavailable requested indices and ignored extras, and keeps strict runner +cardinality. Validator and runner boundary checks passed with +`go test ./internal/framework/validators ./internal/framework/runner`. + **Work** 1. Read the validator models, `llm_validators.go`, existing partial-batch tests, diff --git a/internal/framework/outcomes/outcomes.go b/internal/framework/outcomes/outcomes.go index a58ef09..f887696 100644 --- a/internal/framework/outcomes/outcomes.go +++ b/internal/framework/outcomes/outcomes.go @@ -59,9 +59,10 @@ const ( type RecoveryReason string const ( - RecoveryOutputRepaired RecoveryReason = "output_repaired" - RecoveryServiceRecovered RecoveryReason = "service_recovered" - RecoveryProposalsFiltered RecoveryReason = "proposals_filtered" + RecoveryOutputRepaired RecoveryReason = "output_repaired" + RecoveryServiceRecovered RecoveryReason = "service_recovered" + RecoveryProposalsFiltered RecoveryReason = "proposals_filtered" + RecoveryValidationDecisionsFiltered RecoveryReason = "validation_decisions_filtered" ) // FilteredProposal attributes a defective response item to its proposal identity. diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 12a6c9c..b1d4c50 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -1026,6 +1026,49 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress( } } +func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ + {CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}, + {CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}, + {CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"}, + }}}} + first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) + var laterIndices []int + later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + decisions := make([]validators.Decision, 0, len(req.CandidateProposal)) + for _, candidate := range req.CandidateProposal { + laterIndices = append(laterIndices, candidate.ProposalIndex) + approved := candidate.ProposalIndex == 0 + disposition := validators.DispositionRejected + if approved { + disposition = validators.DispositionApproved + } + decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: approved, Disposition: disposition, ReasonCode: "later_review"}) + } + return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil + }} + module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) { + return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{ + {TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}, + {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}, + {TargetSegmentID: 1, OriginalText: "bad", CorrectedText: "good", Confidence: 1}, + }, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}}, nil + }} + r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}}) + cfg := runnerTestConfig() + out, err := r.Run(context.Background(), RunInput{ + Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}}, + ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, + }) + if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 { + t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err) + } + unit := out.WorkUnits[0] + if unit.CorrectionsApplied != 1 || len(unit.Operations) != 2 || !unit.Operations[0].Succeeded || unit.Operations[1].FailureReason != outcomes.ReasonValidationDecisionsInvalid || len(unit.Operations[1].ProposalIndices) != 1 || unit.Operations[1].ProposalIndices[0] != 1 || len(unit.Operations[1].RecoveryReasons) != 0 { + t.Fatalf("operation accounting: %+v", unit) + } +} + func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) diff --git a/internal/framework/validators/llm_partial_test.go b/internal/framework/validators/llm_partial_test.go index 241bab2..a54e6ea 100644 --- a/internal/framework/validators/llm_partial_test.go +++ b/internal/framework/validators/llm_partial_test.go @@ -95,11 +95,32 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) { req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client result, err := validator.Validate(context.Background(), req) - if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 { + if err != nil || len(result.Decisions) != 1 || result.Decisions[0].ReasonCode != ReasonMissingValidationDecision || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 || !equalIndices(result.Operations[0].ProposalIndices, []int{0}) { t.Fatalf("domain-invalid repaired response: %+v, %v", result, err) } } +func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) { + client := &scriptedValidationClient{replies: []validationReply{{ + response: LLMValidationResponse{Validations: []LLMValidationDecision{ + {CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}, + {CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"}, + }}, + completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}}, + }}} + validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) + req.LLMClient = client + result, err := validator.Validate(context.Background(), req) + if err != nil || len(result.Decisions) != 1 || !result.Decisions[0].Approved || len(result.Operations) != 1 { + t.Fatalf("validation result: %+v, %v", result, err) + } + op := result.Operations[0] + if !op.Succeeded || op.Counts.ProviderAttempts != 3 || op.Counts.TransientRetries != 1 || op.Counts.StructuralRepairs != 1 || len(op.RecoveryReasons) != 3 || op.RecoveryReasons[0] != outcomes.RecoveryOutputRepaired || op.RecoveryReasons[1] != outcomes.RecoveryServiceRecovered || op.RecoveryReasons[2] != outcomes.RecoveryValidationDecisionsFiltered || len(op.Filtered) != 0 { + t.Fatalf("operation: %+v", op) + } +} + func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) { cause := errors.New("broken embedded schema") client := &scriptedValidationClient{replies: []validationReply{ diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 4687d2f..6328a62 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -149,10 +149,12 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, Counts: completion.Execution, } - artifacts := InteractionArtifacts{} - if req.DiagnosticsWriter != nil { + writeInteraction := func(responsePayload any, callErr error) InteractionArtifacts { + if req.DiagnosticsWriter == nil { + return InteractionArtifacts{} + } var diagnosticErr error - artifacts, diagnosticErr = req.DiagnosticsWriter.WriteInteraction( + artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction( stage, map[string]any{ "validator_name": v.name, @@ -162,14 +164,16 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, "response_schema": responseSchema.DiagnosticsMap(), }, completion.Prepared, - map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, - errPayload(err), + responsePayload, + errPayload(callErr), ) if diagnosticErr != nil && req.ReportDiagnosticError != nil { req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr)) } + return artifacts } if err != nil { + writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, err) var failure *structuredoutput.Failure if errors.As(err, &failure) { operation.FailureReason = failure.Reason @@ -186,26 +190,28 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) } - batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response) - if err != nil { + reconciled := reconcileLLMResponse(batch.Items, response) + artifacts := writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model, "reconciliation": reconciled}, nil) + if len(reconciled.UnavailableIndices) > 0 { operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid - operation.ProposalIndices = batchProposalIndices(batch.Items) - operations = append(operations, operation) - llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(outcomes.ReasonValidationDecisionsInvalid), "model validation decisions were unusable")...) - continue - } - operation.Succeeded = true - if completion.Execution.StructuralRepairs > 0 { - operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired) - } - if completion.Execution.TransientRetries > 0 { - operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered) + operation.ProposalIndices = reconciled.UnavailableIndices + } else { + operation.Succeeded = true + if completion.Execution.StructuralRepairs > 0 { + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired) + } + if completion.Execution.TransientRetries > 0 { + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered) + } + if len(reconciled.IgnoredIndices) > 0 { + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryValidationDecisionsFiltered) + } } operations = append(operations, operation) - for i := range batchDecisions { - batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath + for i := range reconciled.Decisions { + reconciled.Decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath } - llmDecisions = append(llmDecisions, batchDecisions...) + llmDecisions = append(llmDecisions, reconciled.Decisions...) } all := append([]Decision(nil), immediate...) @@ -288,40 +294,50 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT }, immediate } -func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) { - expected := make(map[int]LLMValidationItem, len(items)) - for _, item := range items { - expected[item.CorrectionIndex] = item - } - if len(response.Validations) == 0 { - return nil, fmt.Errorf("missing validations in structured response") - } +type validationReconciliation struct { + Decisions []Decision `json:"decisions"` + UnavailableIndices []int `json:"unavailable_indices,omitempty"` + IgnoredIndices []int `json:"ignored_indices,omitempty"` +} - seen := make(map[int]LLMValidationDecision, len(response.Validations)) +func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation { + expected := make(map[int]struct{}, len(items)) + for _, item := range items { + expected[item.CorrectionIndex] = struct{}{} + } + result := validationReconciliation{Decisions: make([]Decision, 0, len(items))} + seen := make(map[int][]LLMValidationDecision, len(items)) for _, d := range response.Validations { - if d.Confidence < 0.0 || d.Confidence > 1.0 { - return nil, fmt.Errorf("confidence for correction_index %d must be between 0.0 and 1.0", d.CorrectionIndex) - } if _, ok := expected[d.CorrectionIndex]; !ok { - return nil, fmt.Errorf("unknown correction_index %d", d.CorrectionIndex) + result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex) + continue } - if _, exists := seen[d.CorrectionIndex]; exists { - return nil, fmt.Errorf("duplicate correction_index %d", d.CorrectionIndex) - } - seen[d.CorrectionIndex] = d + seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d) } - decisions := make([]Decision, 0, len(items)) for _, item := range items { - d, ok := seen[item.CorrectionIndex] - if !ok { - return nil, fmt.Errorf("missing correction_index %d", item.CorrectionIndex) + entries := seen[item.CorrectionIndex] + if len(entries) == 0 { + result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing")) + result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex) + continue + } + if len(entries) > 1 { + result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonDuplicateValidationDecision, "required model validation decision appeared more than once")) + result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex) + continue + } + d := entries[0] + if d.Confidence < 0.0 || d.Confidence > 1.0 { + result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonInvalidValidationDecision, "required model validation decision had invalid confidence")) + result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex) + continue } reasonCode := ReasonApproved if !d.Approved { reasonCode = "llm_rejected" } - decisions = append(decisions, Decision{ + result.Decisions = append(result.Decisions, Decision{ ProposalIndex: item.CorrectionIndex, Approved: d.Approved, Disposition: decisionDisposition(d.Approved), @@ -329,7 +345,7 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation Message: strings.TrimSpace(d.Reason), }) } - return decisions, nil + return result } func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision { diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index 73d4d1f..1ef087a 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -26,14 +26,15 @@ type fakeStructuredLLMClient struct { type captureValidationDiagnosticsWriter struct { err error lastRequestMetadata any + lastResponsePayload any } func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) { _ = stage _ = requestPayload - _ = responsePayload _ = errorPayload w.lastRequestMetadata = requestMetadata + w.lastResponsePayload = responsePayload return InteractionArtifacts{}, w.err } @@ -203,49 +204,82 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) { } } -func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) { - client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}} - v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) - req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) - req.LLMClient = client - res, err := v.Validate(context.Background(), req) - if err != nil { - t.Fatalf("expected missing decision downgrade, got %v", err) +func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) { + approve := func(index int) LLMValidationDecision { + return LLMValidationDecision{CorrectionIndex: index, Approved: true, Confidence: .9, Reason: "accepted"} } - if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable { - t.Fatalf("unexpected decisions: %+v", res.Decisions) + reject := func(index int) LLMValidationDecision { + return LLMValidationDecision{CorrectionIndex: index, Confidence: .9, Reason: "editorial concern"} + } + type expectedDecision struct { + index int + approved bool + reason string + } + for _, tc := range []struct { + name string + entries []LLMValidationDecision + want []expectedDecision + unavailable []int + ignored []int + }{ + {"missing and extra", []LLMValidationDecision{approve(7), reject(18), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}}, + {"conflicting duplicate", []LLMValidationDecision{approve(7), approve(11), reject(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, + {"identical duplicate", []LLMValidationDecision{approve(7), approve(11), approve(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, + {"extra only", []LLMValidationDecision{approve(99), reject(18), approve(7), approve(99), approve(11)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}}, + {"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil}, + {"reordered", []LLMValidationDecision{reject(18), approve(11), approve(7)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil}, + {"invalid confidence", []LLMValidationDecision{approve(7), {CorrectionIndex: 11, Approved: true, Confidence: 1.2}, reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}} + writer := &captureValidationDiagnosticsWriter{} + v, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "actors"), mk(18, "gestures", "figures")}) + req.LLMClient = client + req.DiagnosticsWriter = writer + result, err := v.Validate(context.Background(), req) + if err != nil || len(client.calls) != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 { + t.Fatalf("result=%+v calls=%d err=%v", result, len(client.calls), err) + } + for i, want := range tc.want { + got := result.Decisions[i] + if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) { + t.Fatalf("decision[%d]=%+v want=%+v", i, got, want) + } + if got.ReasonCode == "llm_rejected" && got.Message != "editorial concern" { + t.Fatalf("negative reason lost: %+v", got) + } + } + op := result.Operations[0] + if op.Succeeded != (len(tc.unavailable) == 0) || !equalIndices(op.ProposalIndices, tc.unavailable) || (op.FailureReason == outcomes.ReasonValidationDecisionsInvalid) != (len(tc.unavailable) > 0) || len(op.Filtered) != 0 { + t.Fatalf("operation=%+v", op) + } + if len(tc.unavailable) > 0 && len(op.RecoveryReasons) != 0 || len(tc.unavailable) == 0 && (len(op.RecoveryReasons) == 1) != (len(tc.ignored) > 0) { + t.Fatalf("recovery=%+v", op) + } + if len(tc.ignored) > 0 && len(tc.unavailable) == 0 && op.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered { + t.Fatalf("recovery reason=%+v", op) + } + payload := writer.lastResponsePayload.(map[string]any) + reconciliation := payload["reconciliation"].(validationReconciliation) + if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) { + t.Fatalf("diagnostic reconciliation=%+v", reconciliation) + } + }) } } -func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) { - client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ - {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, - {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, - }}}} - v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) - req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) - req.LLMClient = client - res, err := v.Validate(context.Background(), req) - if err != nil { - t.Fatalf("expected duplicate decision downgrade, got %v", err) +func equalIndices(a, b []int) bool { + if len(a) != len(b) { + return false } - if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable { - t.Fatalf("unexpected decisions: %+v", res.Decisions) - } -} - -func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) { - client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}} - v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) - req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) - req.LLMClient = client - res, err := v.Validate(context.Background(), req) - if err != nil { - t.Fatalf("expected unknown index downgrade, got %v", err) - } - if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable { - t.Fatalf("unexpected decisions: %+v", res.Decisions) + for i := range a { + if a[i] != b[i] { + return false + } } + return true } func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) { diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 1a4185e..d5225d1 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -13,18 +13,21 @@ import ( ) const ( - ReasonApproved = "approved" - ReasonLowConfidence = "low_confidence" - ReasonMissingOriginalText = "missing_original_text" - ReasonMissingTargetSegment = "missing_target_segment" - ReasonEmptyResultingText = "empty_resulting_segment" - ReasonNoEffect = "no_effect" - ReasonProtectedGlossaryTerm = "protected_glossary_term" - ReasonInvalidTargetSegment = "invalid_target_segment_id" - ReasonEmptyOriginalText = "empty_original_text" - ReasonInvalidConfidence = "invalid_confidence" - ReasonValidatorMalformed = "validator_response_malformed" - ReasonValidatorInputTooLarge = "validator_input_too_large" + ReasonApproved = "approved" + ReasonLowConfidence = "low_confidence" + ReasonMissingOriginalText = "missing_original_text" + ReasonMissingTargetSegment = "missing_target_segment" + ReasonEmptyResultingText = "empty_resulting_segment" + ReasonNoEffect = "no_effect" + ReasonProtectedGlossaryTerm = "protected_glossary_term" + ReasonInvalidTargetSegment = "invalid_target_segment_id" + ReasonEmptyOriginalText = "empty_original_text" + ReasonInvalidConfidence = "invalid_confidence" + ReasonValidatorMalformed = "validator_response_malformed" + ReasonValidatorInputTooLarge = "validator_input_too_large" + ReasonMissingValidationDecision = "missing_validation_decision" + ReasonDuplicateValidationDecision = "duplicate_validation_decision" + ReasonInvalidValidationDecision = "invalid_validation_decision" ) // Request is the runtime input shared by deterministic validators.