eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Preserve validation decisions across failed batches

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:14:33 +00:00
parent 42bde59827
commit 0a19f163c6
7 changed files with 233 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/roadmap/implementation.md
View File

@@ -495,6 +495,16 @@ as recovery.
## Stage 4 — Preserve Successful Validation Batches
**Completed 2026-09-24.** LLM validation now keeps successful batch decisions
around recoverable failed batches, emits explicit unavailable decisions for failed
or oversized inputs, and records one compact operation per client call plus one
consolidated oversized-input operation. Invalid decision sets fail their own
batch, and retry/repair recovery is recorded only after usable domain mapping.
Fatal failures retain earlier facts and stop later batches. Focused validator,
proposal, and four-module tests pass, as do validator/proposal race tests.
`git diff --check` passed. Runner compilation is still deferred because its old
warning fields are reconnected in Stage 6.
**Work:** Refactor `LLMBackedValidator.Validate`, input preparation, and response
mapping per D. Collect all successful batch decisions; on a recoverable failure
add non-approved unavailable decisions for that batch and continue. Keep invalid

4
internal/framework/proposal_generation/generate_test.go
View File

@@ -372,8 +372,8 @@ func TestGenerateCandidatesClientError(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "completion failed") {
t.Fatalf("expected completion failure, got %v", err)
}
if got.Artifacts.ResponsePayloadPath != "" {
t.Fatalf("expected zero result on error, got %+v", got)
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
}
matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json"))
if globErr != nil {

24
internal/framework/validators/llm_integration_test.go
View File

@@ -13,20 +13,22 @@ import (
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
)
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":0,"approved":true,"confidence":0.9,"reason":"ok"}]}`
for _, tc := range []struct {
name string
responses []string
wantCalls int
wantApproved, wantWarning, wantRuntime bool
name string
responses []string
wantCalls int
wantApproved bool
wantFailure outcomes.ReasonCode
}{
{"repair reaches decisions", []string{`{"wrong":true}`, valid}, 2, true, false, false},
{"exhausted repair rejects batch", []string{`{"wrong":true}`, `{"wrong":true}`}, 2, false, true, false},
{"malformed envelope propagates", []string{"MALFORMED_ENVELOPE"}, 1, false, false, true},
{"repair reaches decisions", []string{`{"wrong":true}`, valid}, 2, true, ""},
{"exhausted repair rejects batch", []string{`{"wrong":true}`, `{"wrong":true}`}, 2, false, outcomes.ReasonInvalidOutputExhausted},
{"malformed envelope excludes batch", []string{"MALFORMED_ENVELOPE"}, 1, false, outcomes.ReasonProviderResponseInvalid},
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
@@ -79,16 +81,10 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
if calls != tc.wantCalls {
t.Fatalf("calls=%d, want %d", calls, tc.wantCalls)
}
if tc.wantRuntime {
if err == nil || len(result.Warnings) != 0 {
t.Fatalf("runtime failure downgraded: %+v, %v", result, err)
}
return
}
if err != nil {
t.Fatal(err)
}
if len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.wantApproved || (len(result.Warnings) > 0) != tc.wantWarning {
if len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.wantApproved || len(result.Operations) != 1 || result.Operations[0].FailureReason != tc.wantFailure {
t.Fatalf("domain result: %+v", result)
}
})

119
internal/framework/validators/llm_partial_test.go Normal file
View File

@@ -0,0 +1,119 @@
package validators
import (
"context"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
)
type perItemEstimator struct{}
func (perItemEstimator) EstimateTokens(payload string) int {
return 100 * strings.Count(payload, `"correction_index"`)
}
type validationReply struct {
response LLMValidationResponse
completion StructuredCompletionResponse
err error
}
type scriptedValidationClient struct {
replies []validationReply
calls []StructuredCompletionRequest
}
func (c *scriptedValidationClient) CompleteStructured(_ context.Context, req StructuredCompletionRequest, target any) (StructuredCompletionResponse, error) {
c.calls = append(c.calls, req)
if len(c.replies) == 0 {
return StructuredCompletionResponse{}, errors.New("unexpected validation call")
}
reply := c.replies[0]
c.replies = c.replies[1:]
if reply.err == nil {
*target.(*LLMValidationResponse) = reply.response
}
return reply.completion, reply.err
}
func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{
mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"),
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
req := makeReq(candidates)
req.Config.Validation.MaxPromptTokens = 150
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if err != nil {
t.Fatal(err)
}
if len(result.Decisions) != 3 || !result.Decisions[0].Approved || result.Decisions[1].Disposition != DispositionUnavailable || !result.Decisions[2].Approved {
t.Fatalf("batch decisions: %+v", result.Decisions)
}
if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 {
t.Fatalf("batch operations: %+v", result.Operations)
}
if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil {
t.Fatal(err)
}
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
nextReq := makeReq(survivors)
nextReq.Config.Validation.MaxPromptTokens = 150
nextReq.LLMClient = nextClient
next, err := nextValidator.Validate(context.Background(), nextReq)
if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 {
t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err)
}
}
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
}}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 {
t.Fatalf("domain-invalid repaired response: %+v, %v", result, err)
}
}
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures")})
req.Config.Validation.MaxPromptTokens = 150
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if !errors.Is(err, cause) || len(result.Decisions) != 1 || len(result.Operations) != 2 || len(client.calls) != 2 {
t.Fatalf("fatal partial result: %+v, calls=%d, err=%v", result, len(client.calls), err)
}
}

121
internal/framework/validators/llm_validators.go
View File

@@ -2,6 +2,7 @@ package validators
import (
"context"
"errors"
"fmt"
"sort"
"strings"
@@ -9,11 +10,11 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
@@ -74,33 +75,58 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
maxTokens = req.Config.Validation.MaxPromptTokens
}
warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
operations := make([]outcomes.Operation, 0)
sectionIndex := 0
if req.SectionIndex != nil {
sectionIndex = *req.SectionIndex
}
operationRef := func(kind outcomes.OperationKind, batchIndex *int) outcomes.Reference {
return outcomes.Reference{
ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance,
SectionIndex: sectionIndex, Kind: kind, ValidatorName: v.name,
ValidatorOrder: req.ValidatorOrder, BatchIndex: batchIndex,
}
}
if len(oversized) > 0 {
indices := make([]int, len(oversized))
for i, decision := range oversized {
indices[i] = decision.ProposalIndex
}
operations = append(operations, outcomes.Operation{
Reference: operationRef(outcomes.OperationValidationInput, nil),
FailureReason: outcomes.ReasonValidationInputTooLarge,
ProposalIndices: indices,
})
}
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
if len(itemsForBatching) == 0 {
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
}
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
if err != nil {
return Result{}, err
return Result{ValidatorName: v.name, Decisions: append(append([]Decision(nil), immediate...), oversized...), Operations: operations}, err
}
llmDecisions := make([]Decision, 0)
for _, batch := range batches {
if err := ctx.Err(); err != nil {
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
}
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.Context.Description
}
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
if err != nil {
return Result{}, err
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
}
var response LLMValidationResponse
@@ -117,6 +143,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
}, &response)
batchIndex := batch.BatchIndex
operation := outcomes.Operation{
Reference: operationRef(outcomes.OperationValidation, &batchIndex),
Counts: completion.Execution,
}
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
@@ -135,20 +166,38 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
)
}
if err != nil {
if structuredoutput.IsMalformedError(err) {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
var failure *structuredoutput.Failure
if errors.As(err, &failure) {
operation.FailureReason = failure.Reason
}
if operation.FailureReason == "" {
operation.FailureReason = outcomes.ReasonInternalFailure
}
operation.ProposalIndices = batchProposalIndices(batch.Items)
operations = append(operations, operation)
if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...)
continue
}
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
}
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
if err != nil {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid
operation.ProposalIndices = batchProposalIndices(batch.Items)
operations = append(operations, operation)
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(outcomes.ReasonValidationDecisionsInvalid), "model validation decisions were unusable")...)
continue
}
operation.Succeeded = true
if completion.Execution.StructuralRepairs > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
}
if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
}
operations = append(operations, operation)
for i := range batchDecisions {
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
}
@@ -159,10 +208,10 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
all = append(all, oversized...)
all = append(all, llmDecisions...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
}
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
@@ -271,6 +320,7 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
decisions = append(decisions, Decision{
ProposalIndex: item.CorrectionIndex,
Approved: d.Approved,
Disposition: decisionDisposition(d.Approved),
ReasonCode: reasonCode,
Message: strings.TrimSpace(d.Reason),
})
@@ -285,24 +335,7 @@ func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
}
return out
}
func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
out := make([]stagewarnings.StageWarning, 0)
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
ReasonCode: ReasonValidatorInputTooLarge,
Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
})
out = append(out, unavailable(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
}
return out
}
@@ -328,26 +361,22 @@ func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LL
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
out := make([]Decision, 0, len(items))
for _, item := range items {
out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
out = append(out, unavailable(item.CorrectionIndex, reasonCode, message))
}
return out
}
func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
idx := batchIndex
return stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
BatchIndex: &idx,
ReasonCode: reasonCode,
Message: strings.TrimSpace(message),
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
func batchProposalIndices(items []LLMValidationItem) []int {
indices := make([]int, len(items))
for i, item := range items {
indices[i] = item.CorrectionIndex
}
return indices
}
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
if artifacts.ErrorPayloadPath != "" {
return artifacts.ErrorPayloadPath
func decisionDisposition(approved bool) DecisionDisposition {
if approved {
return DispositionApproved
}
return artifacts.ResponsePayloadPath
return DispositionRejected
}

23
internal/framework/validators/llm_validators_test.go
View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
@@ -136,7 +137,7 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
}
func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}}
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
@@ -144,11 +145,11 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
if err != nil {
t.Fatalf("expected malformed output downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable || res.Decisions[0].ReasonCode != string(outcomes.ReasonInvalidOutputExhausted) {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
if len(res.Operations) != 1 || res.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
t.Fatalf("expected failed operation, got %+v", res.Operations)
}
}
@@ -158,7 +159,7 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
result, err := v.Validate(context.Background(), req)
if err == nil || len(result.Warnings) != 0 {
if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonInternalFailure {
t.Fatalf("protocol error was downgraded: %+v, %v", result, err)
}
}
@@ -172,7 +173,7 @@ func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
@@ -189,7 +190,7 @@ func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
@@ -203,7 +204,7 @@ func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
if err != nil {
t.Fatalf("expected unknown index downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
}
@@ -245,14 +246,14 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
if len(res.Decisions) != 2 {
t.Fatalf("expected two decisions, got %+v", res.Decisions)
}
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved {
if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable {
t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
}
if !res.Decisions[1].Approved {
t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("expected one oversize warning, got %+v", res.Warnings)
if len(res.Operations) != 2 || res.Operations[0].Reference.Kind != outcomes.OperationValidationInput || res.Operations[0].FailureReason != outcomes.ReasonValidationInputTooLarge || res.Operations[0].Reference.BatchIndex != nil || len(res.Operations[0].ProposalIndices) != 1 || res.Operations[0].ProposalIndices[0] != 0 {
t.Fatalf("expected consolidated oversized input outcome, got %+v", res.Operations)
}
}

5
internal/framework/validators/models.go
View File

@@ -34,6 +34,7 @@ type Request struct {
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ValidatorOrder *int `json:"validator_order,omitempty"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Glossary *schema.Glossary `json:"-"`
Config *config.Config `json:"-"`
@@ -120,6 +121,10 @@ func rejection(index int, reasonCode string, msg string) Decision {
return Decision{ProposalIndex: index, Approved: false, Disposition: DispositionRejected, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)}
}
func unavailable(index int, reasonCode string, msg string) Decision {
return Decision{ProposalIndex: index, Approved: false, Disposition: DispositionUnavailable, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)}
}
func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64 {
if cfg == nil {
return 0.0