Hardened the proposal modules to skip malformed proposals rather than hard failing the entire run
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
This commit is contained in:
@@ -143,10 +143,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
||||
}
|
||||
|
||||
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err)
|
||||
}
|
||||
batchDecisions := mapLLMResponseToDecisions(batch.Items, response)
|
||||
for i := range batchDecisions {
|
||||
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
||||
}
|
||||
@@ -259,34 +256,48 @@ func promptBuilderForType(validatorType LLMValidatorType) (LLMPromptBuilder, err
|
||||
}
|
||||
}
|
||||
|
||||
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) {
|
||||
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) []Decision {
|
||||
expected := make(map[int]LLMValidationItem, len(items))
|
||||
for _, item := range items {
|
||||
expected[item.CorrectionIndex] = item
|
||||
}
|
||||
if len(response.Validations) == 0 {
|
||||
return nil, fmt.Errorf("missing validations in structured response")
|
||||
}
|
||||
|
||||
seen := make(map[int]LLMValidationDecision, len(response.Validations))
|
||||
forcedReject := make(map[int]bool)
|
||||
for _, d := range response.Validations {
|
||||
if d.Confidence < 0.0 || d.Confidence > 1.0 {
|
||||
return nil, fmt.Errorf("confidence for correction_index %d must be between 0.0 and 1.0", d.CorrectionIndex)
|
||||
}
|
||||
if _, ok := expected[d.CorrectionIndex]; !ok {
|
||||
return nil, fmt.Errorf("unknown correction_index %d", d.CorrectionIndex)
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[d.CorrectionIndex]; exists {
|
||||
return nil, fmt.Errorf("duplicate correction_index %d", d.CorrectionIndex)
|
||||
forcedReject[d.CorrectionIndex] = true
|
||||
continue
|
||||
}
|
||||
if d.Confidence < 0.0 || d.Confidence > 1.0 {
|
||||
forcedReject[d.CorrectionIndex] = true
|
||||
continue
|
||||
}
|
||||
seen[d.CorrectionIndex] = d
|
||||
}
|
||||
|
||||
decisions := make([]Decision, 0, len(items))
|
||||
for _, item := range items {
|
||||
if forcedReject[item.CorrectionIndex] {
|
||||
decisions = append(decisions, Decision{
|
||||
ProposalIndex: item.CorrectionIndex,
|
||||
Approved: false,
|
||||
ReasonCode: ReasonValidatorMalformed,
|
||||
Message: "validator returned malformed decision payload for this proposal index",
|
||||
})
|
||||
continue
|
||||
}
|
||||
d, ok := seen[item.CorrectionIndex]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("missing correction_index %d", item.CorrectionIndex)
|
||||
decisions = append(decisions, Decision{
|
||||
ProposalIndex: item.CorrectionIndex,
|
||||
Approved: false,
|
||||
ReasonCode: ReasonValidatorMissing,
|
||||
Message: "validator did not return a decision for this proposal index",
|
||||
})
|
||||
continue
|
||||
}
|
||||
reasonCode := ReasonApproved
|
||||
if !d.Approved {
|
||||
@@ -299,5 +310,5 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
|
||||
Message: strings.TrimSpace(d.Reason),
|
||||
})
|
||||
}
|
||||
return decisions, nil
|
||||
return decisions
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user