421 lines
15 KiB
Go
421 lines
15 KiB
Go
package validators
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
|
stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings"
|
|
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
|
)
|
|
|
|
type LLMPromptBuilder func(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error)
|
|
|
|
type LLMBackedValidator struct {
|
|
name string
|
|
validatorType LLMValidatorType
|
|
promptBuilder LLMPromptBuilder
|
|
model string
|
|
estimator chunking.TokenEstimator
|
|
}
|
|
|
|
func (v *LLMBackedValidator) Name() string {
|
|
return v.name
|
|
}
|
|
|
|
// SetTokenEstimator allows deterministic test control over batching behavior.
|
|
func (v *LLMBackedValidator) SetTokenEstimator(estimator chunking.TokenEstimator) {
|
|
if v == nil || estimator == nil {
|
|
return
|
|
}
|
|
v.estimator = estimator
|
|
}
|
|
|
|
func NewLLMBackedValidator(name string, validatorType LLMValidatorType, model string) (*LLMBackedValidator, error) {
|
|
builder, err := promptBuilderForType(validatorType)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if strings.TrimSpace(name) == "" {
|
|
return nil, fmt.Errorf("validator name must not be empty")
|
|
}
|
|
return &LLMBackedValidator{
|
|
name: name,
|
|
validatorType: validatorType,
|
|
promptBuilder: builder,
|
|
model: strings.TrimSpace(model),
|
|
estimator: chunking.NewSimpleTokenEstimator(),
|
|
}, nil
|
|
}
|
|
|
|
func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, error) {
|
|
if v == nil {
|
|
return Result{}, fmt.Errorf("validator is nil")
|
|
}
|
|
if req.LLMClient == nil {
|
|
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
|
|
}
|
|
if len(req.CandidateProposal) == 0 {
|
|
return Result{ValidatorName: v.name, Decisions: nil}, nil
|
|
}
|
|
validationReq, immediate := BuildLLMValidationRequest(v.name, v.validatorType, req)
|
|
if len(validationReq.Items) == 0 {
|
|
all := append([]Decision(nil), immediate...)
|
|
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
|
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
|
return Result{}, err
|
|
}
|
|
return Result{ValidatorName: v.name, Decisions: all}, nil
|
|
}
|
|
|
|
maxTokens := config.DefaultValidationMaxPromptTokens
|
|
if req.Config != nil && req.Config.ValidationMaxPromptTokens > 0 {
|
|
maxTokens = req.Config.ValidationMaxPromptTokens
|
|
}
|
|
|
|
warnings := append([]stagewarnings.StageWarning(nil), oversizedValidationWarnings(v.name, maxTokens, validationReq.Items, v.estimator)...)
|
|
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
|
|
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
|
|
if len(itemsForBatching) == 0 {
|
|
all := append([]Decision(nil), immediate...)
|
|
all = append(all, oversized...)
|
|
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
|
return Result{}, err
|
|
}
|
|
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
|
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
|
|
}
|
|
|
|
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
|
|
llmDecisions := make([]Decision, 0)
|
|
for _, batch := range batches {
|
|
transcriptDescription := ""
|
|
if req.Config != nil {
|
|
transcriptDescription = req.Config.TranscriptDescription
|
|
}
|
|
messages, err := v.promptBuilder(batch.Items, transcriptDescription)
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
|
|
var response LLMValidationResponse
|
|
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
|
call := func(callCtx context.Context) error {
|
|
_, err = req.LLMClient.CompleteStructured(callCtx, StructuredCompletionRequest{
|
|
StageName: fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex),
|
|
Messages: messages,
|
|
Model: resolvedValidationModel(req.Config, v.model),
|
|
ResponseSchema: &responseSchema,
|
|
}, &response)
|
|
return err
|
|
}
|
|
if req.Scheduler != nil {
|
|
err = req.Scheduler.Run(ctx, call)
|
|
} else {
|
|
err = call(ctx)
|
|
}
|
|
artifacts := InteractionArtifacts{}
|
|
if req.DiagnosticsWriter != nil {
|
|
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
|
|
promptMetadata := validatorPromptMetadata(v.validatorType)
|
|
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
|
|
stage,
|
|
map[string]any{
|
|
"validator_name": v.name,
|
|
"validator_type": v.validatorType,
|
|
"batch_index": batch.BatchIndex,
|
|
"prompt_metadata": map[string]any{
|
|
"prompt_id": promptMetadata.PromptID,
|
|
"prompt_version": promptMetadata.PromptVersion,
|
|
"prompt_source": promptMetadata.PromptSource,
|
|
"embedded_path": promptMetadata.EmbeddedPath,
|
|
"sha256": promptMetadata.SHA256,
|
|
},
|
|
"response_schema": map[string]any{
|
|
"id": responseSchema.ID,
|
|
"version": responseSchema.Version,
|
|
"name": responseSchema.Name,
|
|
"sha256": responseSchema.SHA256,
|
|
},
|
|
},
|
|
map[string]any{"messages": messages, "items": batch.Items},
|
|
response,
|
|
errPayload(err),
|
|
)
|
|
}
|
|
if err != nil {
|
|
if isMalformedStructuredOutputError(err) {
|
|
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
|
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
|
continue
|
|
}
|
|
return Result{}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
|
}
|
|
|
|
batchDecisions, err := mapLLMResponseToDecisions(batch.Items, response)
|
|
if err != nil {
|
|
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...)
|
|
warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts))
|
|
continue
|
|
}
|
|
for i := range batchDecisions {
|
|
batchDecisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
|
}
|
|
llmDecisions = append(llmDecisions, batchDecisions...)
|
|
}
|
|
|
|
all := append([]Decision(nil), immediate...)
|
|
all = append(all, oversized...)
|
|
all = append(all, llmDecisions...)
|
|
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
|
return Result{}, err
|
|
}
|
|
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
|
return Result{ValidatorName: v.name, Decisions: all, Warnings: warnings}, nil
|
|
}
|
|
|
|
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
|
|
switch validatorType {
|
|
case LLMValidatorTypeSpokenFormPlausibility:
|
|
return prompts.MustLookupMetadata(prompts.PromptIDValidatorSpokenFormPlausibility)
|
|
case LLMValidatorTypeMeaningReversal:
|
|
return prompts.MustLookupMetadata(prompts.PromptIDValidatorMeaningReversalReview)
|
|
case LLMValidatorTypeEditorialReview:
|
|
return prompts.MustLookupMetadata(prompts.PromptIDValidatorEditorialReview)
|
|
case LLMValidatorTypeGrammarReview:
|
|
return prompts.MustLookupMetadata(prompts.PromptIDValidatorGrammarReview)
|
|
case LLMValidatorTypeSpokenWordReview:
|
|
return prompts.MustLookupMetadata(prompts.PromptIDValidatorSpokenWordReview)
|
|
default:
|
|
return prompts.Metadata{}
|
|
}
|
|
}
|
|
|
|
func errPayload(err error) any {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
return map[string]any{"error": err.Error()}
|
|
}
|
|
|
|
func resolvedValidationModel(cfg *config.Config, override string) string {
|
|
if strings.TrimSpace(override) != "" {
|
|
return strings.TrimSpace(override)
|
|
}
|
|
if cfg == nil {
|
|
return ""
|
|
}
|
|
return cfg.EffectiveValidationLLMConfig().Model
|
|
}
|
|
|
|
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
|
|
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
|
|
immediate := make([]Decision, 0)
|
|
segments := make(map[int]schema.Segment)
|
|
if req.WorkingTranscript != nil {
|
|
segments = make(map[int]schema.Segment, len(req.WorkingTranscript.Segments))
|
|
for _, seg := range req.WorkingTranscript.Segments {
|
|
segments[seg.ID] = seg
|
|
}
|
|
}
|
|
|
|
for _, p := range req.CandidateProposal {
|
|
seg, ok := segments[p.TargetSegmentID]
|
|
if !ok {
|
|
immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found"))
|
|
continue
|
|
}
|
|
|
|
preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy)
|
|
if !preview.Applicable {
|
|
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
|
|
continue
|
|
}
|
|
|
|
items = append(items, LLMValidationItem{
|
|
CorrectionIndex: p.ProposalIndex,
|
|
SegmentID: p.TargetSegmentID,
|
|
OriginalText: p.OriginalText,
|
|
CorrectedText: p.CorrectedText,
|
|
OriginalSegmentText: seg.Text,
|
|
CorrectedSegmentText: preview.CorrectedSegmentText,
|
|
Categories: append([]string(nil), seg.Categories...),
|
|
})
|
|
}
|
|
|
|
return LLMValidationRequest{
|
|
ValidatorName: validatorName,
|
|
ValidatorType: validatorType,
|
|
ModuleKey: req.ModuleKey,
|
|
ModuleInstance: req.ModuleInstance,
|
|
ReplacementPolicy: string(req.ReplacementPolicy),
|
|
Glossary: req.Glossary,
|
|
Items: items,
|
|
}, immediate
|
|
}
|
|
|
|
func promptBuilderForType(validatorType LLMValidatorType) (LLMPromptBuilder, error) {
|
|
switch validatorType {
|
|
case LLMValidatorTypeSpokenFormPlausibility:
|
|
return BuildSpokenFormPlausibilityMessages, nil
|
|
case LLMValidatorTypeMeaningReversal:
|
|
return BuildMeaningReversalMessages, nil
|
|
case LLMValidatorTypeEditorialReview:
|
|
return BuildEditorialMessages, nil
|
|
case LLMValidatorTypeGrammarReview:
|
|
return BuildGrammarReviewMessages, nil
|
|
case LLMValidatorTypeSpokenWordReview:
|
|
return BuildSpokenWordReviewMessages, nil
|
|
default:
|
|
return nil, fmt.Errorf("unsupported LLM validator type %q", validatorType)
|
|
}
|
|
}
|
|
|
|
func mapLLMResponseToDecisions(items []LLMValidationItem, response LLMValidationResponse) ([]Decision, error) {
|
|
expected := make(map[int]LLMValidationItem, len(items))
|
|
for _, item := range items {
|
|
expected[item.CorrectionIndex] = item
|
|
}
|
|
if len(response.Validations) == 0 {
|
|
return nil, fmt.Errorf("missing validations in structured response")
|
|
}
|
|
|
|
seen := make(map[int]LLMValidationDecision, len(response.Validations))
|
|
for _, d := range response.Validations {
|
|
if d.Confidence < 0.0 || d.Confidence > 1.0 {
|
|
return nil, fmt.Errorf("confidence for correction_index %d must be between 0.0 and 1.0", d.CorrectionIndex)
|
|
}
|
|
if _, ok := expected[d.CorrectionIndex]; !ok {
|
|
return nil, fmt.Errorf("unknown correction_index %d", d.CorrectionIndex)
|
|
}
|
|
if _, exists := seen[d.CorrectionIndex]; exists {
|
|
return nil, fmt.Errorf("duplicate correction_index %d", d.CorrectionIndex)
|
|
}
|
|
seen[d.CorrectionIndex] = d
|
|
}
|
|
|
|
decisions := make([]Decision, 0, len(items))
|
|
for _, item := range items {
|
|
d, ok := seen[item.CorrectionIndex]
|
|
if !ok {
|
|
return nil, fmt.Errorf("missing correction_index %d", item.CorrectionIndex)
|
|
}
|
|
reasonCode := ReasonApproved
|
|
if !d.Approved {
|
|
reasonCode = "llm_rejected"
|
|
}
|
|
decisions = append(decisions, Decision{
|
|
ProposalIndex: item.CorrectionIndex,
|
|
Approved: d.Approved,
|
|
ReasonCode: reasonCode,
|
|
Message: strings.TrimSpace(d.Reason),
|
|
})
|
|
}
|
|
return decisions, nil
|
|
}
|
|
|
|
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
|
|
out := make([]Decision, 0)
|
|
for _, item := range items {
|
|
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
|
if err != nil || singleTokens <= maxPromptTokens {
|
|
continue
|
|
}
|
|
out = append(out, rejection(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func oversizedValidationWarnings(validatorName string, maxPromptTokens int, items []LLMValidationItem, estimator chunking.TokenEstimator) []stagewarnings.StageWarning {
|
|
out := make([]stagewarnings.StageWarning, 0)
|
|
for _, item := range items {
|
|
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
|
if err != nil || singleTokens <= maxPromptTokens {
|
|
continue
|
|
}
|
|
out = append(out, stagewarnings.StageWarning{
|
|
Scope: stagewarnings.ScopeValidator,
|
|
ValidatorName: validatorName,
|
|
ReasonCode: ReasonValidatorInputTooLarge,
|
|
Message: fmt.Sprintf("validation input exceeds max prompt tokens for proposal %d", item.CorrectionIndex),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
|
|
if len(decisions) == 0 {
|
|
return append([]LLMValidationItem(nil), items...)
|
|
}
|
|
rejected := make(map[int]struct{}, len(decisions))
|
|
for _, decision := range decisions {
|
|
rejected[decision.ProposalIndex] = struct{}{}
|
|
}
|
|
out := make([]LLMValidationItem, 0, len(items))
|
|
for _, item := range items {
|
|
if _, ok := rejected[item.CorrectionIndex]; ok {
|
|
continue
|
|
}
|
|
out = append(out, item)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
|
|
out := make([]Decision, 0, len(items))
|
|
for _, item := range items {
|
|
out = append(out, rejection(item.CorrectionIndex, reasonCode, message))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func newValidatorWarning(validatorName string, batchIndex int, reasonCode string, message string, artifacts InteractionArtifacts) stagewarnings.StageWarning {
|
|
idx := batchIndex
|
|
return stagewarnings.StageWarning{
|
|
Scope: stagewarnings.ScopeValidator,
|
|
ValidatorName: validatorName,
|
|
BatchIndex: &idx,
|
|
ReasonCode: reasonCode,
|
|
Message: strings.TrimSpace(message),
|
|
DiagnosticArtifactPath: diagnosticArtifactPath(artifacts),
|
|
}
|
|
}
|
|
|
|
func diagnosticArtifactPath(artifacts InteractionArtifacts) string {
|
|
if artifacts.ErrorPayloadPath != "" {
|
|
return artifacts.ErrorPayloadPath
|
|
}
|
|
return artifacts.ResponsePayloadPath
|
|
}
|
|
|
|
func isMalformedStructuredOutputError(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
msg := err.Error()
|
|
for _, marker := range []string{
|
|
"malformed structured output",
|
|
"decode structured output:",
|
|
"decode provider response envelope:",
|
|
"provider response missing choices",
|
|
"provider response missing assistant message content",
|
|
"provider response assistant message content is empty",
|
|
"provider response assistant message content is not valid JSON",
|
|
} {
|
|
if strings.Contains(msg, marker) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|