eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Reconcile partial validator decision responses

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 11:25:22 +00:00
parent 7199cb565f
commit d726ea8f6d
7 changed files with 221 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
internal/framework/validators/llm_validators.go
View File

@@ -149,10 +149,12 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
Counts: completion.Execution,
}
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
writeInteraction := func(responsePayload any, callErr error) InteractionArtifacts {
if req.DiagnosticsWriter == nil {
return InteractionArtifacts{}
}
var diagnosticErr error
artifacts, diagnosticErr = req.DiagnosticsWriter.WriteInteraction(
artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"validator_name": v.name,
@@ -162,14 +164,16 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
"response_schema": responseSchema.DiagnosticsMap(),
},
completion.Prepared,
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
errPayload(err),
responsePayload,
errPayload(callErr),
)
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr))
}
return artifacts
}
if err != nil {
writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}, err)
var failure *structuredoutput.Failure
if errors.As(err, &failure) {
operation.FailureReason = failure.Reason
@@ -186,26 +190,28 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
}
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
if err != nil {
reconciled := reconcileLLMResponse(batch.Items, response)
artifacts := writeInteraction(map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model, "reconciliation": reconciled}, nil)
if len(reconciled.UnavailableIndices) > 0 {
operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid
operation.ProposalIndices = batchProposalIndices(batch.Items)
operations = append(operations, operation)
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(outcomes.ReasonValidationDecisionsInvalid), "model validation decisions were unusable")...)
continue
}
operation.Succeeded = true
if completion.Execution.StructuralRepairs > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
}
if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
operation.ProposalIndices = reconciled.UnavailableIndices
} else {
operation.Succeeded = true
if completion.Execution.StructuralRepairs > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
}
if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
}
if len(reconciled.IgnoredIndices) > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryValidationDecisionsFiltered)
}
}
operations = append(operations, operation)
for i := range batchDecisions {
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
for i := range reconciled.Decisions {
reconciled.Decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
}
llmDecisions = append(llmDecisions, batchDecisions...)
llmDecisions = append(llmDecisions, reconciled.Decisions...)
}
all := append([]Decision(nil), immediate...)
@@ -288,40 +294,50 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
}, immediate
}
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) {
expected := make(map[int]LLMValidationItem, len(items))
for _, item := range items {
expected[item.CorrectionIndex] = item
}
if len(response.Validations) == 0 {
return nil, fmt.Errorf("missing validations in structured response")
}
type validationReconciliation struct {
Decisions []Decision `json:"decisions"`
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
IgnoredIndices []int `json:"ignored_indices,omitempty"`
}
seen := make(map[int]LLMValidationDecision, len(response.Validations))
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
expected := make(map[int]struct{}, len(items))
for _, item := range items {
expected[item.CorrectionIndex] = struct{}{}
}
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
seen := make(map[int][]LLMValidationDecision, len(items))
for _, d := range response.Validations {
if d.Confidence < 0.0 || d.Confidence > 1.0 {
return nil, fmt.Errorf("confidence for correction_index %d must be between 0.0 and 1.0", d.CorrectionIndex)
}
if _, ok := expected[d.CorrectionIndex]; !ok {
return nil, fmt.Errorf("unknown correction_index %d", d.CorrectionIndex)
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
continue
}
if _, exists := seen[d.CorrectionIndex]; exists {
return nil, fmt.Errorf("duplicate correction_index %d", d.CorrectionIndex)
}
seen[d.CorrectionIndex] = d
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
}
decisions := make([]Decision, 0, len(items))
for _, item := range items {
d, ok := seen[item.CorrectionIndex]
if !ok {
return nil, fmt.Errorf("missing correction_index %d", item.CorrectionIndex)
entries := seen[item.CorrectionIndex]
if len(entries) == 0 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
continue
}
if len(entries) > 1 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonDuplicateValidationDecision, "required model validation decision appeared more than once"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
continue
}
d := entries[0]
if d.Confidence < 0.0 || d.Confidence > 1.0 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonInvalidValidationDecision, "required model validation decision had invalid confidence"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
continue
}
reasonCode := ReasonApproved
if !d.Approved {
reasonCode = "llm_rejected"
}
decisions = append(decisions, Decision{
result.Decisions = append(result.Decisions, Decision{
ProposalIndex: item.CorrectionIndex,
Approved: d.Approved,
Disposition: decisionDisposition(d.Approved),
@@ -329,7 +345,7 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
Message: strings.TrimSpace(d.Reason),
})
}
return decisions, nil
return result
}
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {