eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Preserve validation decisions across failed batches

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:14:33 +00:00
parent 42bde59827
commit 0a19f163c6
7 changed files with 233 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
internal/framework/validators/llm_validators.go
View File

@@ -2,6 +2,7 @@ package validators
import (
"context"
"errors"
"fmt"
"sort"
"strings"
@@ -9,11 +10,11 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
@@ -74,33 +75,58 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
maxTokens = req.Config.Validation.MaxPromptTokens
}
warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
operations := make([]outcomes.Operation, 0)
sectionIndex := 0
if req.SectionIndex != nil {
sectionIndex = *req.SectionIndex
}
operationRef := func(kind outcomes.OperationKind, batchIndex *int) outcomes.Reference {
return outcomes.Reference{
ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance,
SectionIndex: sectionIndex, Kind: kind, ValidatorName: v.name,
ValidatorOrder: req.ValidatorOrder, BatchIndex: batchIndex,
}
}
if len(oversized) > 0 {
indices := make([]int, len(oversized))
for i, decision := range oversized {
indices[i] = decision.ProposalIndex
}
operations = append(operations, outcomes.Operation{
Reference: operationRef(outcomes.OperationValidationInput, nil),
FailureReason: outcomes.ReasonValidationInputTooLarge,
ProposalIndices: indices,
})
}
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
if len(itemsForBatching) == 0 {
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
}
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
if err != nil {
return Result{}, err
return Result{ValidatorName: v.name, Decisions: append(append([]Decision(nil), immediate...), oversized...), Operations: operations}, err
}
llmDecisions := make([]Decision, 0)
for _, batch := range batches {
if err := ctx.Err(); err != nil {
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
}
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.Context.Description
}
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
if err != nil {
return Result{}, err
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
}
var response LLMValidationResponse
@@ -117,6 +143,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
}, &response)
batchIndex := batch.BatchIndex
operation := outcomes.Operation{
Reference: operationRef(outcomes.OperationValidation, &batchIndex),
Counts: completion.Execution,
}
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
@@ -135,20 +166,38 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
)
}
if err != nil {
if structuredoutput.IsMalformedError(err) {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
var failure *structuredoutput.Failure
if errors.As(err, &failure) {
operation.FailureReason = failure.Reason
}
if operation.FailureReason == "" {
operation.FailureReason = outcomes.ReasonInternalFailure
}
operation.ProposalIndices = batchProposalIndices(batch.Items)
operations = append(operations, operation)
if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...)
continue
}
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
}
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
if err != nil {
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid
operation.ProposalIndices = batchProposalIndices(batch.Items)
operations = append(operations, operation)
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(outcomes.ReasonValidationDecisionsInvalid), "model validation decisions were unusable")...)
continue
}
operation.Succeeded = true
if completion.Execution.StructuralRepairs > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
}
if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
}
operations = append(operations, operation)
for i := range batchDecisions {
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
}
@@ -159,10 +208,10 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
all = append(all, oversized...)
all = append(all, llmDecisions...)
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{}, err
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
}
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
@@ -271,6 +320,7 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
decisions = append(decisions, Decision{
ProposalIndex: item.CorrectionIndex,
Approved: d.Approved,
Disposition: decisionDisposition(d.Approved),
ReasonCode: reasonCode,
Message: strings.TrimSpace(d.Reason),
})
@@ -285,24 +335,7 @@ func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
}
return out
}
func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
out := make([]stagewarnings.StageWarning, 0)
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
ReasonCode: ReasonValidatorInputTooLarge,
Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
})
out = append(out, unavailable(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
}
return out
}
@@ -328,26 +361,22 @@ func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LL
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
out := make([]Decision, 0, len(items))
for _, item := range items {
out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
out = append(out, unavailable(item.CorrectionIndex, reasonCode, message))
}
return out
}
func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
idx := batchIndex
return stagewarnings.StageWarning{
Scope: stagewarnings.ScopeValidator,
ValidatorName: validatorName,
BatchIndex: &idx,
ReasonCode: reasonCode,
Message: strings.TrimSpace(message),
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
func batchProposalIndices(items []LLMValidationItem) []int {
indices := make([]int, len(items))
for i, item := range items {
indices[i] = item.CorrectionIndex
}
return indices
}
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
if artifacts.ErrorPayloadPath != "" {
return artifacts.ErrorPayloadPath
func decisionDisposition(approved bool) DecisionDisposition {
if approved {
return DispositionApproved
}
return artifacts.ResponsePayloadPath
return DispositionRejected
}