From 0a19f163c6e68cfdab04562a9404a8efc7e8ce36 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 24 Sep 2026 17:14:33 +0000 Subject: [PATCH] Preserve validation decisions across failed batches --- docs/roadmap/implementation.md | 10 ++ .../proposal_generation/generate_test.go | 4 +- .../validators/llm_integration_test.go | 24 ++-- .../framework/validators/llm_partial_test.go | 119 +++++++++++++++++ .../framework/validators/llm_validators.go | 121 +++++++++++------- .../validators/llm_validators_test.go | 23 ++-- internal/framework/validators/models.go | 5 + 7 files changed, 233 insertions(+), 73 deletions(-) create mode 100644 internal/framework/validators/llm_partial_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index bc4bf7d..0601292 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -495,6 +495,16 @@ as recovery. ## Stage 4 — Preserve Successful Validation Batches +**Completed 2026-09-24.** LLM validation now keeps successful batch decisions +around recoverable failed batches, emits explicit unavailable decisions for failed +or oversized inputs, and records one compact operation per client call plus one +consolidated oversized-input operation. Invalid decision sets fail their own +batch, and retry/repair recovery is recorded only after usable domain mapping. +Fatal failures retain earlier facts and stop later batches. Focused validator, +proposal, and four-module tests pass, as do validator/proposal race tests. +`git diff --check` passed. Runner compilation is still deferred because its old +warning fields are reconnected in Stage 6. + **Work:** Refactor `LLMBackedValidator.Validate`, input preparation, and response mapping per D. Collect all successful batch decisions; on a recoverable failure add non-approved unavailable decisions for that batch and continue. Keep invalid diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index 114ea86..be4f329 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -372,8 +372,8 @@ func TestGenerateCandidatesClientError(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "completion failed") { t.Fatalf("expected completion failure, got %v", err) } - if got.Artifacts.ResponsePayloadPath != "" { - t.Fatalf("expected zero result on error, got %+v", got) + if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 { + t.Fatalf("expected fatal call facts without proposals, got %+v", got) } matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json")) if globErr != nil { diff --git a/internal/framework/validators/llm_integration_test.go b/internal/framework/validators/llm_integration_test.go index e90360e..212bffa 100644 --- a/internal/framework/validators/llm_integration_test.go +++ b/internal/framework/validators/llm_integration_test.go @@ -13,20 +13,22 @@ import ( ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/framework/llm" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" ) func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { valid := `{"validations":[{"correction_index":0,"approved":true,"confidence":0.9,"reason":"ok"}]}` for _, tc := range []struct { - name string - responses []string - wantCalls int - wantApproved, wantWarning, wantRuntime bool + name string + responses []string + wantCalls int + wantApproved bool + wantFailure outcomes.ReasonCode }{ - {"repair reaches decisions", []string{`{"wrong":true}`, valid}, 2, true, false, false}, - {"exhausted repair rejects batch", []string{`{"wrong":true}`, `{"wrong":true}`}, 2, false, true, false}, - {"malformed envelope propagates", []string{"MALFORMED_ENVELOPE"}, 1, false, false, true}, + {"repair reaches decisions", []string{`{"wrong":true}`, valid}, 2, true, ""}, + {"exhausted repair rejects batch", []string{`{"wrong":true}`, `{"wrong":true}`}, 2, false, outcomes.ReasonInvalidOutputExhausted}, + {"malformed envelope excludes batch", []string{"MALFORMED_ENVELOPE"}, 1, false, outcomes.ReasonProviderResponseInvalid}, } { t.Run(tc.name, func(t *testing.T) { calls := 0 @@ -79,16 +81,10 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { if calls != tc.wantCalls { t.Fatalf("calls=%d, want %d", calls, tc.wantCalls) } - if tc.wantRuntime { - if err == nil || len(result.Warnings) != 0 { - t.Fatalf("runtime failure downgraded: %+v, %v", result, err) - } - return - } if err != nil { t.Fatal(err) } - if len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.wantApproved || (len(result.Warnings) > 0) != tc.wantWarning { + if len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.wantApproved || len(result.Operations) != 1 || result.Operations[0].FailureReason != tc.wantFailure { t.Fatalf("domain result: %+v", result) } }) diff --git a/internal/framework/validators/llm_partial_test.go b/internal/framework/validators/llm_partial_test.go new file mode 100644 index 0000000..241bab2 --- /dev/null +++ b/internal/framework/validators/llm_partial_test.go @@ -0,0 +1,119 @@ +package validators + +import ( + "context" + "errors" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" +) + +type perItemEstimator struct{} + +func (perItemEstimator) EstimateTokens(payload string) int { + return 100 * strings.Count(payload, `"correction_index"`) +} + +type validationReply struct { + response LLMValidationResponse + completion StructuredCompletionResponse + err error +} + +type scriptedValidationClient struct { + replies []validationReply + calls []StructuredCompletionRequest +} + +func (c *scriptedValidationClient) CompleteStructured(_ context.Context, req StructuredCompletionRequest, target any) (StructuredCompletionResponse, error) { + c.calls = append(c.calls, req) + if len(c.replies) == 0 { + return StructuredCompletionResponse{}, errors.New("unexpected validation call") + } + reply := c.replies[0] + c.replies = c.replies[1:] + if reply.err == nil { + *target.(*LLMValidationResponse) = reply.response + } + return reply.completion, reply.err +} + +func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { + candidates := []proposals.EnrichedCorrectionProposal{ + mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"), + } + failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"} + client := &scriptedValidationClient{replies: []validationReply{ + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, + {completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, + }} + validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) + validator.SetTokenEstimator(perItemEstimator{}) + req := makeReq(candidates) + req.Config.Validation.MaxPromptTokens = 150 + req.LLMClient = client + result, err := validator.Validate(context.Background(), req) + if err != nil { + t.Fatal(err) + } + if len(result.Decisions) != 3 || !result.Decisions[0].Approved || result.Decisions[1].Disposition != DispositionUnavailable || !result.Decisions[2].Approved { + t.Fatalf("batch decisions: %+v", result.Decisions) + } + if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 { + t.Fatalf("batch operations: %+v", result.Operations) + } + if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil { + t.Fatal(err) + } + + survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]} + nextClient := &scriptedValidationClient{replies: []validationReply{ + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, + }} + nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal) + nextValidator.SetTokenEstimator(perItemEstimator{}) + nextReq := makeReq(survivors) + nextReq.Config.Validation.MaxPromptTokens = 150 + nextReq.LLMClient = nextClient + next, err := nextValidator.Validate(context.Background(), nextReq) + if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 { + t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err) + } +} + +func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) { + client := &scriptedValidationClient{replies: []validationReply{{ + response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}}, + completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}}, + }}} + validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) + req.LLMClient = client + result, err := validator.Validate(context.Background(), req) + if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Disposition != DispositionUnavailable || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid || result.Operations[0].Counts.StructuralRepairs != 1 || len(result.Operations[0].RecoveryReasons) != 0 { + t.Fatalf("domain-invalid repaired response: %+v, %v", result, err) + } +} + +func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) { + cause := errors.New("broken embedded schema") + client := &scriptedValidationClient{replies: []validationReply{ + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, + {err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, + }} + validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) + validator.SetTokenEstimator(perItemEstimator{}) + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures")}) + req.Config.Validation.MaxPromptTokens = 150 + req.LLMClient = client + result, err := validator.Validate(context.Background(), req) + if !errors.Is(err, cause) || len(result.Decisions) != 1 || len(result.Operations) != 2 || len(client.calls) != 2 { + t.Fatalf("fatal partial result: %+v, calls=%d, err=%v", result, len(client.calls), err) + } +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 4923389..1f672bf 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -2,6 +2,7 @@ package validators import ( "context" + "errors" "fmt" "sort" "strings" @@ -9,11 +10,11 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" - stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" "gitea.maximumdirect.net/eric/audita/internal/prompts" ) @@ -74,33 +75,58 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, maxTokens = req.Config.Validation.MaxPromptTokens } - warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...) oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator) + operations := make([]outcomes.Operation, 0) + sectionIndex := 0 + if req.SectionIndex != nil { + sectionIndex = *req.SectionIndex + } + operationRef := func(kind outcomes.OperationKind, batchIndex *int) outcomes.Reference { + return outcomes.Reference{ + ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, + SectionIndex: sectionIndex, Kind: kind, ValidatorName: v.name, + ValidatorOrder: req.ValidatorOrder, BatchIndex: batchIndex, + } + } + if len(oversized) > 0 { + indices := make([]int, len(oversized)) + for i, decision := range oversized { + indices[i] = decision.ProposalIndex + } + operations = append(operations, outcomes.Operation{ + Reference: operationRef(outcomes.OperationValidationInput, nil), + FailureReason: outcomes.ReasonValidationInputTooLarge, + ProposalIndices: indices, + }) + } itemsForBatching := filterItemsByDecision(validationReq.Items, oversized) if len(itemsForBatching) == 0 { all := append([]Decision(nil), immediate...) all = append(all, oversized...) if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { - return Result{}, err + return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err } sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) - return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil + return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil } batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator) if err != nil { - return Result{}, err + return Result{ValidatorName: v.name, Decisions: append(append([]Decision(nil), immediate...), oversized...), Operations: operations}, err } llmDecisions := make([]Decision, 0) for _, batch := range batches { + if err := ctx.Err(); err != nil { + return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err + } transcriptDescription := "" if req.Config != nil { transcriptDescription = req.Config.Context.Description } variables, err := BuildValidationVariables(batch.Items, transcriptDescription) if err != nil { - return Result{}, err + return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err } var response LLMValidationResponse @@ -117,6 +143,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, Variables: variables, Schema: responseschema.ValidatorDecisionSetKey, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name}, }, &response) + batchIndex := batch.BatchIndex + operation := outcomes.Operation{ + Reference: operationRef(outcomes.OperationValidation, &batchIndex), + Counts: completion.Execution, + } artifacts := InteractionArtifacts{} if req.DiagnosticsWriter != nil { @@ -135,20 +166,38 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, ) } if err != nil { - if structuredoutput.IsMalformedError(err) { - llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...) - warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts)) + var failure *structuredoutput.Failure + if errors.As(err, &failure) { + operation.FailureReason = failure.Reason + } + if operation.FailureReason == "" { + operation.FailureReason = outcomes.ReasonInternalFailure + } + operation.ProposalIndices = batchProposalIndices(batch.Items) + operations = append(operations, operation) + if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" { + llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...) continue } - return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) + return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) } batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response) if err != nil { - llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...) - warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts)) + operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid + operation.ProposalIndices = batchProposalIndices(batch.Items) + operations = append(operations, operation) + llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(outcomes.ReasonValidationDecisionsInvalid), "model validation decisions were unusable")...) continue } + operation.Succeeded = true + if completion.Execution.StructuralRepairs > 0 { + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired) + } + if completion.Execution.TransientRetries > 0 { + operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered) + } + operations = append(operations, operation) for i := range batchDecisions { batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath } @@ -159,10 +208,10 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, all = append(all, oversized...) all = append(all, llmDecisions...) if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { - return Result{}, err + return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err } sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) - return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil + return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil } func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata { @@ -271,6 +320,7 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation decisions = append(decisions, Decision{ ProposalIndex: item.CorrectionIndex, Approved: d.Approved, + Disposition: decisionDisposition(d.Approved), ReasonCode: reasonCode, Message: strings.TrimSpace(d.Reason), }) @@ -285,24 +335,7 @@ func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int if err != nil || singleTokens <= maxPromptTokens { continue } - out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens")) - } - return out -} - -func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning { - out := make([]stagewarnings.StageWarning, 0) - for _, item := range items { - singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item}) - if err != nil || singleTokens <= maxPromptTokens { - continue - } - out = append(out, stagewarnings.StageWarning{ - Scope: stagewarnings.ScopeValidator, - ValidatorName: validatorName, - ReasonCode: ReasonValidatorInputTooLarge, - Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex), - }) + out = append(out, unavailable(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens")) } return out } @@ -328,26 +361,22 @@ func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LL func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision { out := make([]Decision, 0, len(items)) for _, item := range items { - out = append(out, rejection(item.CorrectionIndex, reasonCode, message)) + out = append(out, unavailable(item.CorrectionIndex, reasonCode, message)) } return out } -func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning { - idx := batchIndex - return stagewarnings.StageWarning{ - Scope: stagewarnings.ScopeValidator, - ValidatorName: validatorName, - BatchIndex: &idx, - ReasonCode: reasonCode, - Message: strings.TrimSpace(message), - DiagnosticArtifactPath: diagnosticArtifactPath(artifacts), +func batchProposalIndices(items []LLMValidationItem) []int { + indices := make([]int, len(items)) + for i, item := range items { + indices[i] = item.CorrectionIndex } + return indices } -func diagnosticArtifactPath(artifacts InteractionArtifacts) string { - if artifacts.ErrorPayloadPath != "" { - return artifacts.ErrorPayloadPath +func decisionDisposition(approved bool) DecisionDisposition { + if approved { + return DispositionApproved } - return artifacts.ResponsePayloadPath + return DispositionRejected } diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index 665836a..2220a83 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -9,6 +9,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/schema" + "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" @@ -136,7 +137,7 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) { } func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) { - client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}} + client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}} v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client @@ -144,11 +145,11 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) { if err != nil { t.Fatalf("expected malformed output downgrade, got %v", err) } - if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable || res.Decisions[0].ReasonCode != string(outcomes.ReasonInvalidOutputExhausted) { t.Fatalf("unexpected decisions: %+v", res.Decisions) } - if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed { - t.Fatalf("expected malformed warning, got %+v", res.Warnings) + if len(res.Operations) != 1 || res.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted { + t.Fatalf("expected failed operation, got %+v", res.Operations) } } @@ -158,7 +159,7 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) { req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client result, err := v.Validate(context.Background(), req) - if err == nil || len(result.Warnings) != 0 { + if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonInternalFailure { t.Fatalf("protocol error was downgraded: %+v, %v", result, err) } } @@ -172,7 +173,7 @@ func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) { if err != nil { t.Fatalf("expected missing decision downgrade, got %v", err) } - if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable { t.Fatalf("unexpected decisions: %+v", res.Decisions) } } @@ -189,7 +190,7 @@ func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) { if err != nil { t.Fatalf("expected duplicate decision downgrade, got %v", err) } - if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable { t.Fatalf("unexpected decisions: %+v", res.Decisions) } } @@ -203,7 +204,7 @@ func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) { if err != nil { t.Fatalf("expected unknown index downgrade, got %v", err) } - if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable { t.Fatalf("unexpected decisions: %+v", res.Decisions) } } @@ -245,14 +246,14 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes if len(res.Decisions) != 2 { t.Fatalf("expected two decisions, got %+v", res.Decisions) } - if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved { + if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable { t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0]) } if !res.Decisions[1].Approved { t.Fatalf("expected second decision approved, got %+v", res.Decisions[1]) } - if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorInputTooLarge { - t.Fatalf("expected one oversize warning, got %+v", res.Warnings) + if len(res.Operations) != 2 || res.Operations[0].Reference.Kind != outcomes.OperationValidationInput || res.Operations[0].FailureReason != outcomes.ReasonValidationInputTooLarge || res.Operations[0].Reference.BatchIndex != nil || len(res.Operations[0].ProposalIndices) != 1 || res.Operations[0].ProposalIndices[0] != 0 { + t.Fatalf("expected consolidated oversized input outcome, got %+v", res.Operations) } } diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 866c51b..59f47c7 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -34,6 +34,7 @@ type Request struct { CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"` ModuleKey string `json:"module_key"` ModuleInstance string `json:"module_instance"` + ValidatorOrder *int `json:"validator_order,omitempty"` ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` Glossary *schema.Glossary `json:"-"` Config *config.Config `json:"-"` @@ -120,6 +121,10 @@ func rejection(index int, reasonCode string, msg string) Decision { return Decision{ProposalIndex: index, Approved: false, Disposition: DispositionRejected, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)} } +func unavailable(index int, reasonCode string, msg string) Decision { + return Decision{ProposalIndex: index, Approved: false, Disposition: DispositionUnavailable, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)} +} + func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64 { if cfg == nil { return 0.0