Hardened the proposal modules to skip malformed proposals rather than hard failing the entire run
All checks were successful
ci/woodpecker/tag/release Pipeline was successful

This commit is contained in:
2026-05-17 07:19:24 -05:00
parent a84941d681
commit 9c2d8338d7
17 changed files with 436 additions and 137 deletions

View File

@@ -143,10 +143,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
}
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
if err != nil {
return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err)
}
batchDecisions := mapLLMResponseToDecisions(batch.Items, response)
for i := range batchDecisions {
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
}
@@ -259,34 +256,48 @@ func promptBuilderForType(validatorType LLMValidatorType) (LLMPromptBuilder, err
}
}
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) {
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) []Decision {
expected := make(map[int]LLMValidationItem, len(items))
for _, item := range items {
expected[item.CorrectionIndex] = item
}
if len(response.Validations) == 0 {
return nil, fmt.Errorf("missing validations in structured response")
}
seen := make(map[int]LLMValidationDecision, len(response.Validations))
forcedReject := make(map[int]bool)
for _, d := range response.Validations {
if d.Confidence < 0.0 || d.Confidence > 1.0 {
return nil, fmt.Errorf("confidence for correction_index %d must be between 0.0 and 1.0", d.CorrectionIndex)
}
if _, ok := expected[d.CorrectionIndex]; !ok {
return nil, fmt.Errorf("unknown correction_index %d", d.CorrectionIndex)
continue
}
if _, exists := seen[d.CorrectionIndex]; exists {
return nil, fmt.Errorf("duplicate correction_index %d", d.CorrectionIndex)
forcedReject[d.CorrectionIndex] = true
continue
}
if d.Confidence < 0.0 || d.Confidence > 1.0 {
forcedReject[d.CorrectionIndex] = true
continue
}
seen[d.CorrectionIndex] = d
}
decisions := make([]Decision, 0, len(items))
for _, item := range items {
if forcedReject[item.CorrectionIndex] {
decisions = append(decisions, Decision{
ProposalIndex: item.CorrectionIndex,
Approved: false,
ReasonCode: ReasonValidatorMalformed,
Message: "validator returned malformed decision payload for this proposal index",
})
continue
}
d, ok := seen[item.CorrectionIndex]
if !ok {
return nil, fmt.Errorf("missing correction_index %d", item.CorrectionIndex)
decisions = append(decisions, Decision{
ProposalIndex: item.CorrectionIndex,
Approved: false,
ReasonCode: ReasonValidatorMissing,
Message: "validator did not return a decision for this proposal index",
})
continue
}
reasonCode := ReasonApproved
if !d.Approved {
@@ -299,5 +310,5 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
Message: strings.TrimSpace(d.Reason),
})
}
return decisions, nil
return decisions
}