eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Share prompt instructions and simplify validation helpers

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 13:26:51 +00:00
parent 5feec1bf14
commit 07a4b65570
20 changed files with 172 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
internal/framework/validators/llm_validators.go
View File

@@ -80,6 +80,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
operations := make([]outcomes.Operation, 0)
llmDecisions := make([]Decision, 0)
partialResult := func() Result {
decisions := append([]Decision(nil), immediate...)
decisions = append(decisions, oversized...)
decisions = append(decisions, llmDecisions...)
return Result{ValidatorName: v.name, Decisions: decisions, Operations: operations}
}
sectionIndex := 0
if req.SectionIndex != nil {
sectionIndex = *req.SectionIndex
@@ -104,24 +111,23 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
if len(itemsForBatching) == 0 {
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
result := partialResult()
all := result.Decisions
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
return result, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
return result, nil
}
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
if err != nil {
return Result{ValidatorName: v.name, Decisions: append(append([]Decision(nil), immediate...), oversized...), Operations: operations}, err
return partialResult(), err
}
llmDecisions := make([]Decision, 0)
for _, batch := range batches {
if err := ctx.Err(); err != nil {
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
return partialResult(), err
}
transcriptDescription := ""
if req.Config != nil {
@@ -129,7 +135,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
if err != nil {
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
return partialResult(), err
}
var response LLMValidationResponse
@@ -191,7 +197,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...)
continue
}
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
}
reconciled := reconcileLLMResponse(batch.Items, response)
@@ -201,12 +207,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
operation.ProposalIndices = reconciled.UnavailableIndices
} else {
operation.Succeeded = true
if completion.Execution.StructuralRepairs > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
}
if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
}
operation.RecoveryReasons = outcomes.ExecutionRecoveries(completion.Execution)
if len(reconciled.IgnoredIndices) > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryValidationDecisionsFiltered)
}
@@ -218,14 +219,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
llmDecisions = append(llmDecisions, reconciled.Decisions...)
}
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
all = append(all, llmDecisions...)
result := partialResult()
all := result.Decisions
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
return result, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
return result, nil
}
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
@@ -251,28 +251,31 @@ func errPayload(err error) any {
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
immediate := make([]Decision, 0)
segments := make(map[int]schema.Segment)
var segments []schema.Segment
positions := make(map[int]int)
vocabulary := NewProtectedVocabulary(req.Glossary)
if req.WorkingTranscript != nil {
segments = make(map[int]schema.Segment, len(req.WorkingTranscript.Segments))
for _, seg := range req.WorkingTranscript.Segments {
segments[seg.ID] = seg
segments = req.WorkingTranscript.Segments
for i, seg := range segments {
positions[seg.ID] = i
}
}
for _, p := range req.CandidateProposal {
seg, ok := segments[p.TargetSegmentID]
position, ok := positions[p.TargetSegmentID]
if !ok {
immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found"))
continue
}
seg := segments[position]
preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy)
if !preview.Applicable {
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
continue
}
neighbors, glossaryEntries := validationContext(req, seg.ID, preview.CorrectedSegmentText)
neighbors, glossaryEntries := validationContext(segments, position, req.Glossary, vocabulary, preview.CorrectedSegmentText)
items = append(items, LLMValidationItem{
CorrectionIndex: p.ProposalIndex,
SegmentID: p.TargetSegmentID,