Make module-stage LLM handling resilient and report warnings

internal/framework/validators/llm_validators.go

@@ -11,6 +11,7 @@ import (
 	"gitea.maximumdirect.net/eric/audita/internal/core/schema"
 	"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
 	"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
+	stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
 	"gitea.maximumdirect.net/eric/audita/internal/prompts"
 )
 
@@ -78,7 +79,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
 		maxTokens = req.Config.ValidationMaxPromptTokens
 	}
 
-	batches, err := ChunkLLMValidationItems(validationReq.Items, maxTokens, v.estimator)
+	warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
+	oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
+	itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
+	if len(itemsForBatching) == 0 {
+		all := append([]Decision(nil), immediate...)
+		all = append(all, oversized...)
+		if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
+			return Result{}, err
+		}
+		sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
+		return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
+	}
+
+	batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
 	if err != nil {
 		return Result{}, err
 	}
@@ -140,12 +154,19 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
 			)
 		}
 		if err != nil {
+			if isMalformedStructuredOutputError(err) {
+				llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
+				warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
+				continue
+			}
 			return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
 		}
 
 		batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
 		if err != nil {
-			return Result{}, fmt.Errorf("LLM validator %q response invalid: %w", v.name, err)
+			llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
+			warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
+			continue
 		}
 		for i := range batchDecisions {
 			batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
@@ -154,12 +175,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
 	}
 
 	all := append([]Decision(nil), immediate...)
+	all = append(all, oversized...)
 	all = append(all, llmDecisions...)
 	if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
 		return Result{}, err
 	}
 	sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
-	return Result{ValidatorName: v.name, Decisions: all}, nil
+	return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
 }
 
 func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
@@ -301,3 +323,98 @@ func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidation
 	}
 	return decisions, nil
 }
+
+func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
+	out := make([]Decision, 0)
+	for _, item := range items {
+		singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
+		if err != nil || singleTokens <= maxPromptTokens {
+			continue
+		}
+		out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
+	}
+	return out
+}
+
+func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
+	out := make([]stagewarnings.StageWarning, 0)
+	for _, item := range items {
+		singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
+		if err != nil || singleTokens <= maxPromptTokens {
+			continue
+		}
+		out = append(out, stagewarnings.StageWarning{
+			Scope:         stagewarnings.ScopeValidator,
+			ValidatorName: validatorName,
+			ReasonCode:    ReasonValidatorInputTooLarge,
+			Message:       fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
+		})
+	}
+	return out
+}
+
+func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
+	if len(decisions) == 0 {
+		return append([]LLMValidationItem(nil), items...)
+	}
+	rejected := make(map[int]struct{}, len(decisions))
+	for _, decision := range decisions {
+		rejected[decision.ProposalIndex] = struct{}{}
+	}
+	out := make([]LLMValidationItem, 0, len(items))
+	for _, item := range items {
+		if _, ok := rejected[item.CorrectionIndex]; ok {
+			continue
+		}
+		out = append(out, item)
+	}
+	return out
+}
+
+func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
+	out := make([]Decision, 0, len(items))
+	for _, item := range items {
+		out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
+	}
+	return out
+}
+
+func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
+	idx := batchIndex
+	return stagewarnings.StageWarning{
+		Scope:                  stagewarnings.ScopeValidator,
+		ValidatorName:          validatorName,
+		BatchIndex:             &idx,
+		ReasonCode:             reasonCode,
+		Message:                strings.TrimSpace(message),
+		DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
+	}
+}
+
+func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
+	if artifacts.ErrorPayloadPath != "" {
+		return artifacts.ErrorPayloadPath
+	}
+	return artifacts.ResponsePayloadPath
+}
+
+func isMalformedStructuredOutputError(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	for _, marker := range []string{
+		"malformed structured output",
+		"decode structured output:",
+		"decode provider response envelope:",
+		"provider response missing choices",
+		"provider response missing assistant message content",
+		"provider response assistant message content is empty",
+		"provider response assistant message content is not valid JSON",
+	} {
+		if strings.Contains(msg, marker) {
+			return true
+		}
+	}
+	return false
+}