eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Reconcile partial validator decision responses

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 11:25:22 +00:00
parent 7199cb565f
commit d726ea8f6d
7 changed files with 221 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/roadmap/implementation.md
View File

@@ -143,6 +143,11 @@ cannot accept them as authoritative verdicts.
## Stage 1 — Reconcile Model Decisions and Preserve Valid Work ## Stage 1 — Reconcile Model Decisions and Preserve Valid Work
**Completed:** 2026-09-25. Reconciliation preserves usable decisions, attributes
unavailable requested indices and ignored extras, and keeps strict runner
cardinality. Validator and runner boundary checks passed with
`go test ./internal/framework/validators ./internal/framework/runner`.
**Work** **Work**
1. Read the validator models, `llm_validators.go`, existing partial-batch tests, 1. Read the validator models, `llm_validators.go`, existing partial-batch tests,

7
internal/framework/outcomes/outcomes.go
View File

@@ -59,9 +59,10 @@ const (
type RecoveryReason string type RecoveryReason string
const ( const (
RecoveryOutputRepaired RecoveryReason = "output_repaired" RecoveryOutputRepaired RecoveryReason = "output_repaired"
RecoveryServiceRecovered RecoveryReason = "service_recovered" RecoveryServiceRecovered RecoveryReason = "service_recovered"
RecoveryProposalsFiltered RecoveryReason = "proposals_filtered" RecoveryProposalsFiltered RecoveryReason = "proposals_filtered"
RecoveryValidationDecisionsFiltered RecoveryReason = "validation_decisions_filtered"
) )
// FilteredProposal attributes a defective response item to its proposal identity. // FilteredProposal attributes a defective response item to its proposal identity.

43
internal/framework/runner/runner_test.go
View File

@@ -1026,6 +1026,49 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
} }
} }
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"},
}}}}
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
var laterIndices []int
later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
decisions := make([]validators.Decision, 0, len(req.CandidateProposal))
for _, candidate := range req.CandidateProposal {
laterIndices = append(laterIndices, candidate.ProposalIndex)
approved := candidate.ProposalIndex == 0
disposition := validators.DispositionRejected
if approved {
disposition = validators.DispositionApproved
}
decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: approved, Disposition: disposition, ReasonCode: "later_review"})
}
return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "bad", CorrectedText: "good", Confidence: 1},
}, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client,
})
if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 {
t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err)
}
unit := out.WorkUnits[0]
if unit.CorrectionsApplied != 1 || len(unit.Operations) != 2 || !unit.Operations[0].Succeeded || unit.Operations[1].FailureReason != outcomes.ReasonValidationDecisionsInvalid || len(unit.Operations[1].ProposalIndices) != 1 || unit.Operations[1].ProposalIndices[0] != 1 || len(unit.Operations[1].RecoveryReasons) != 0 {
t.Fatalf("operation accounting: %+v", unit)
}
}
func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) { func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}} client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)

23
internal/framework/validators/llm_partial_test.go
View File

@@ -95,11 +95,32 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client req.LLMClient = client
result, err := validator.Validate(context.Background(), req) result, err := validator.Validate(context.Background(), req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 { if err != nil || len(result.Decisions) != 1 || result.Decisions[0].ReasonCode != ReasonMissingValidationDecision || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 || !equalIndices(result.Operations[0].ProposalIndices, []int{0}) {
t.Fatalf("domain-invalid repaired response: %+v, %v", result, err) t.Fatalf("domain-invalid repaired response: %+v, %v", result, err)
} }
} }
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
}}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if err != nil || len(result.Decisions) != 1 || !result.Decisions[0].Approved || len(result.Operations) != 1 {
t.Fatalf("validation result: %+v, %v", result, err)
}
op := result.Operations[0]
if !op.Succeeded || op.Counts.ProviderAttempts != 3 || op.Counts.TransientRetries != 1 || op.Counts.StructuralRepairs != 1 || len(op.RecoveryReasons) != 3 || op.RecoveryReasons[0] != outcomes.RecoveryOutputRepaired || op.RecoveryReasons[1] != outcomes.RecoveryServiceRecovered || op.RecoveryReasons[2] != outcomes.RecoveryValidationDecisionsFiltered || len(op.Filtered) != 0 {
t.Fatalf("operation: %+v", op)
}
}
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) { func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema") cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{ client := &scriptedValidationClient{replies: []validationReply{

104
internal/framework/validators/llm_validators.go
View File

@@ -149,10 +149,12 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
Counts: completion.Execution, Counts: completion.Execution,
} }
artifacts := InteractionArtifacts{} writeInteraction := func(responsePayload any, callErr error) InteractionArtifacts {
if req.DiagnosticsWriter != nil { if req.DiagnosticsWriter == nil {
return InteractionArtifacts{}
}
var diagnosticErr error var diagnosticErr error
artifacts, diagnosticErr = req.DiagnosticsWriter.WriteInteraction( artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
stage, stage,
map[string]any{ map[string]any{
"validator_name": v.name, "validator_name": v.name,
@@ -162,14 +164,16 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
"response_schema": responseSchema.DiagnosticsMap(), "response_schema": responseSchema.DiagnosticsMap(),
}, },
completion.Prepared, completion.Prepared,
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, responsePayload,
errPayload(err), errPayload(callErr),
) )
if diagnosticErr != nil && req.ReportDiagnosticError != nil { if diagnosticErr != nil && req.ReportDiagnosticError != nil {
req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr)) req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr))
} }
return artifacts
} }
if err != nil { if err != nil {
writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, err)
var failure *structuredoutput.Failure var failure *structuredoutput.Failure
if errors.As(err, &failure) { if errors.As(err, &failure) {
operation.FailureReason = failure.Reason operation.FailureReason = failure.Reason
@@ -186,26 +190,28 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
} }
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response) reconciled := reconcileLLMResponse(batch.Items, response)
if err != nil { artifacts := writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model, "reconciliation": reconciled}, nil)
if len(reconciled.UnavailableIndices) > 0 {
operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid
operation.ProposalIndices = batchProposalIndices(batch.Items) operation.ProposalIndices = reconciled.UnavailableIndices
operations = append(operations, operation) } else {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(outcomes.ReasonValidationDecisionsInvalid), "model validation decisions were unusable")...) operation.Succeeded = true
continue if completion.Execution.StructuralRepairs > 0 {
} operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
operation.Succeeded = true }
if completion.Execution.StructuralRepairs > 0 { if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired) operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
} }
if completion.Execution.TransientRetries > 0 { if len(reconciled.IgnoredIndices) > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered) operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryValidationDecisionsFiltered)
}
} }
operations = append(operations, operation) operations = append(operations, operation)
for i := range batchDecisions { for i := range reconciled.Decisions {
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath reconciled.Decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
} }
llmDecisions = append(llmDecisions, batchDecisions...) llmDecisions = append(llmDecisions, reconciled.Decisions...)
} }
all := append([]Decision(nil), immediate...) all := append([]Decision(nil), immediate...)
@@ -288,40 +294,50 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
}, immediate }, immediate
} }
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) { type validationReconciliation struct {
expected := make(map[int]LLMValidationItem, len(items)) Decisions []Decision `json:"decisions"`
for _, item := range items { UnavailableIndices []int `json:"unavailable_indices,omitempty"`
expected[item.CorrectionIndex] = item IgnoredIndices []int `json:"ignored_indices,omitempty"`
} }
if len(response.Validations) == 0 {
return nil, fmt.Errorf("missing validations in structured response")
}
seen := make(map[int]LLMValidationDecision, len(response.Validations)) func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
expected := make(map[int]struct{}, len(items))
for _, item := range items {
expected[item.CorrectionIndex] = struct{}{}
}
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
seen := make(map[int][]LLMValidationDecision, len(items))
for _, d := range response.Validations { for _, d := range response.Validations {
if d.Confidence < 0.0 || d.Confidence > 1.0 {
return nil, fmt.Errorf("confidence for correction_index %d must be between 0.0 and 1.0", d.CorrectionIndex)
}
if _, ok := expected[d.CorrectionIndex]; !ok { if _, ok := expected[d.CorrectionIndex]; !ok {
return nil, fmt.Errorf("unknown correction_index %d", d.CorrectionIndex) result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
continue
} }
if _, exists := seen[d.CorrectionIndex]; exists { seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
return nil, fmt.Errorf("duplicate correction_index %d", d.CorrectionIndex)
}
seen[d.CorrectionIndex] = d
} }
decisions := make([]Decision, 0, len(items))
for _, item := range items { for _, item := range items {
d, ok := seen[item.CorrectionIndex] entries := seen[item.CorrectionIndex]
if !ok { if len(entries) == 0 {
return nil, fmt.Errorf("missing correction_index %d", item.CorrectionIndex) result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
continue
}
if len(entries) > 1 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonDuplicateValidationDecision, "required model validation decision appeared more than once"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
continue
}
d := entries[0]
if d.Confidence < 0.0 || d.Confidence > 1.0 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonInvalidValidationDecision, "required model validation decision had invalid confidence"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
continue
} }
reasonCode := ReasonApproved reasonCode := ReasonApproved
if !d.Approved { if !d.Approved {
reasonCode = "llm_rejected" reasonCode = "llm_rejected"
} }
decisions = append(decisions, Decision{ result.Decisions = append(result.Decisions, Decision{
ProposalIndex: item.CorrectionIndex, ProposalIndex: item.CorrectionIndex,
Approved: d.Approved, Approved: d.Approved,
Disposition: decisionDisposition(d.Approved), Disposition: decisionDisposition(d.Approved),
@@ -329,7 +345,7 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
Message: strings.TrimSpace(d.Reason), Message: strings.TrimSpace(d.Reason),
}) })
} }
return decisions, nil return result
} }
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision { func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {

110
internal/framework/validators/llm_validators_test.go
View File

@@ -26,14 +26,15 @@ type fakeStructuredLLMClient struct {
type captureValidationDiagnosticsWriter struct { type captureValidationDiagnosticsWriter struct {
err error err error
lastRequestMetadata any lastRequestMetadata any
lastResponsePayload any
} }
func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) { func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
_ = stage _ = stage
_ = requestPayload _ = requestPayload
_ = responsePayload
_ = errorPayload _ = errorPayload
w.lastRequestMetadata = requestMetadata w.lastRequestMetadata = requestMetadata
w.lastResponsePayload = responsePayload
return InteractionArtifacts{}, w.err return InteractionArtifacts{}, w.err
} }
@@ -203,49 +204,82 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
} }
} }
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) { func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}} approve := func(index int) LLMValidationDecision {
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) return LLMValidationDecision{CorrectionIndex: index, Approved: true, Confidence: .9, Reason: "accepted"}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
} }
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable { reject := func(index int) LLMValidationDecision {
t.Fatalf("unexpected decisions: %+v", res.Decisions) return LLMValidationDecision{CorrectionIndex: index, Confidence: .9, Reason: "editorial concern"}
}
type expectedDecision struct {
index int
approved bool
reason string
}
for _, tc := range []struct {
name string
entries []LLMValidationDecision
want []expectedDecision
unavailable []int
ignored []int
}{
{"missing and extra", []LLMValidationDecision{approve(7), reject(18), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"conflicting duplicate", []LLMValidationDecision{approve(7), approve(11), reject(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(7), approve(11), approve(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(18), approve(7), approve(99), approve(11)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(18), approve(11), approve(7)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(7), {CorrectionIndex: 11, Approved: true, Confidence: 1.2}, reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
} {
t.Run(tc.name, func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
writer := &captureValidationDiagnosticsWriter{}
v, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "actors"), mk(18, "gestures", "figures")})
req.LLMClient = client
req.DiagnosticsWriter = writer
result, err := v.Validate(context.Background(), req)
if err != nil || len(client.calls) != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
t.Fatalf("result=%+v calls=%d err=%v", result, len(client.calls), err)
}
for i, want := range tc.want {
got := result.Decisions[i]
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) {
t.Fatalf("decision[%d]=%+v want=%+v", i, got, want)
}
if got.ReasonCode == "llm_rejected" && got.Message != "editorial concern" {
t.Fatalf("negative reason lost: %+v", got)
}
}
op := result.Operations[0]
if op.Succeeded != (len(tc.unavailable) == 0) || !equalIndices(op.ProposalIndices, tc.unavailable) || (op.FailureReason == outcomes.ReasonValidationDecisionsInvalid) != (len(tc.unavailable) > 0) || len(op.Filtered) != 0 {
t.Fatalf("operation=%+v", op)
}
if len(tc.unavailable) > 0 && len(op.RecoveryReasons) != 0 || len(tc.unavailable) == 0 && (len(op.RecoveryReasons) == 1) != (len(tc.ignored) > 0) {
t.Fatalf("recovery=%+v", op)
}
if len(tc.ignored) > 0 && len(tc.unavailable) == 0 && op.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered {
t.Fatalf("recovery reason=%+v", op)
}
payload := writer.lastResponsePayload.(map[string]any)
reconciliation := payload["reconciliation"].(validationReconciliation)
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
t.Fatalf("diagnostic reconciliation=%+v", reconciliation)
}
})
} }
} }
func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) { func equalIndices(a, b []int) bool {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ if len(a) != len(b) {
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, return false
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
} }
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable { for i := range a {
t.Fatalf("unexpected decisions: %+v", res.Decisions) if a[i] != b[i] {
} return false
} }
func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected unknown index downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
} }
return true
} }
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) { func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {

27
internal/framework/validators/models.go
View File

@@ -13,18 +13,21 @@ import (
) )
const ( const (
ReasonApproved = "approved" ReasonApproved = "approved"
ReasonLowConfidence = "low_confidence" ReasonLowConfidence = "low_confidence"
ReasonMissingOriginalText = "missing_original_text" ReasonMissingOriginalText = "missing_original_text"
ReasonMissingTargetSegment = "missing_target_segment" ReasonMissingTargetSegment = "missing_target_segment"
ReasonEmptyResultingText = "empty_resulting_segment" ReasonEmptyResultingText = "empty_resulting_segment"
ReasonNoEffect = "no_effect" ReasonNoEffect = "no_effect"
ReasonProtectedGlossaryTerm = "protected_glossary_term" ReasonProtectedGlossaryTerm = "protected_glossary_term"
ReasonInvalidTargetSegment = "invalid_target_segment_id" ReasonInvalidTargetSegment = "invalid_target_segment_id"
ReasonEmptyOriginalText = "empty_original_text" ReasonEmptyOriginalText = "empty_original_text"
ReasonInvalidConfidence = "invalid_confidence" ReasonInvalidConfidence = "invalid_confidence"
ReasonValidatorMalformed = "validator_response_malformed" ReasonValidatorMalformed = "validator_response_malformed"
ReasonValidatorInputTooLarge = "validator_input_too_large" ReasonValidatorInputTooLarge = "validator_input_too_large"
ReasonMissingValidationDecision = "missing_validation_decision"
ReasonDuplicateValidationDecision = "duplicate_validation_decision"
ReasonInvalidValidationDecision = "invalid_validation_decision"
) )
// Request is the runtime input shared by deterministic validators. // Request is the runtime input shared by deterministic validators.