eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Reconcile partial validator decision responses

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 11:25:22 +00:00
parent 7199cb565f
commit d726ea8f6d
7 changed files with 221 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
internal/framework/runner/runner_test.go
View File

@@ -1026,6 +1026,49 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
}
}
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"},
}}}}
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
var laterIndices []int
later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
decisions := make([]validators.Decision, 0, len(req.CandidateProposal))
for _, candidate := range req.CandidateProposal {
laterIndices = append(laterIndices, candidate.ProposalIndex)
approved := candidate.ProposalIndex == 0
disposition := validators.DispositionRejected
if approved {
disposition = validators.DispositionApproved
}
decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: approved, Disposition: disposition, ReasonCode: "later_review"})
}
return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "bad", CorrectedText: "good", Confidence: 1},
}, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client,
})
if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 {
t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err)
}
unit := out.WorkUnits[0]
if unit.CorrectionsApplied != 1 || len(unit.Operations) != 2 || !unit.Operations[0].Succeeded || unit.Operations[1].FailureReason != outcomes.ReasonValidationDecisionsInvalid || len(unit.Operations[1].ProposalIndices) != 1 || unit.Operations[1].ProposalIndices[0] != 1 || len(unit.Operations[1].RecoveryReasons) != 0 {
t.Fatalf("operation accounting: %+v", unit)
}
}
func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)