eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Share prompt instructions and simplify validation helpers

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 13:26:51 +00:00
parent 5feec1bf14
commit 07a4b65570
20 changed files with 172 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
internal/framework/validators/llm_context.go
View File

@@ -8,39 +8,29 @@ import (
// validationContext supplies adjacent source segments within the current chunk
// and glossary definitions mentioned in the candidate or that local context.
// These fields travel in each item so batching accounts for their size.
func validationContext(req Request, segmentID int, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
func validationContext(segments []schema.Segment, position int, glossary *schema.Glossary, vocabulary ProtectedVocabulary, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
var neighbors []NeighboringSegment
var source string
if req.WorkingTranscript != nil {
segments := req.WorkingTranscript.Segments
for i, segment := range segments {
if segment.ID != segmentID {
continue
}
source = segment.Text
if i > 0 {
neighbors = append(neighbors, neighboringSegment("preceding", segments[i-1]))
}
if i+1 < len(segments) {
neighbors = append(neighbors, neighboringSegment("following", segments[i+1]))
}
break
}
source := segments[position].Text
if position > 0 {
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
}
if position+1 < len(segments) {
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
}
texts := []string{source, corrected}
for _, neighbor := range neighbors {
texts = append(texts, neighbor.Text)
}
var entries []schema.GlossaryEntry
if req.Glossary != nil {
vocabulary := NewProtectedVocabulary(req.Glossary)
if glossary != nil {
mentioned := make(map[int]bool)
for _, text := range texts {
for identity := range vocabulary.occurrencesByIdentity(text) {
mentioned[identity] = true
}
}
for i, entry := range req.Glossary.Entries {
for i, entry := range glossary.Entries {
if mentioned[i] {
entries = append(entries, entry)
}

16
internal/framework/validators/llm_context_test.go
View File

@@ -76,3 +76,19 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
t.Fatalf("context must count toward batch budget: %+v %v", batches, err)
}
}
func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Actors"}, {Name: "Jesters"}}}
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if len(immediate) != 0 || len(payload.Items) != 2 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
for i, name := range []string{"Jesters", "Actors"} {
entries := payload.Items[i].GlossaryEntries
if len(entries) != 1 || entries[0].Name != name {
t.Fatalf("candidate %d context: %+v", i, entries)
}
}
}

55
internal/framework/validators/llm_validators.go
View File

@@ -80,6 +80,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
operations := make([]outcomes.Operation, 0)
llmDecisions := make([]Decision, 0)
partialResult := func() Result {
decisions := append([]Decision(nil), immediate...)
decisions = append(decisions, oversized...)
decisions = append(decisions, llmDecisions...)
return Result{ValidatorName: v.name, Decisions: decisions, Operations: operations}
}
sectionIndex := 0
if req.SectionIndex != nil {
sectionIndex = *req.SectionIndex
@@ -104,24 +111,23 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
if len(itemsForBatching) == 0 {
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
result := partialResult()
all := result.Decisions
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
return result, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
return result, nil
}
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
if err != nil {
return Result{ValidatorName: v.name, Decisions: append(append([]Decision(nil), immediate...), oversized...), Operations: operations}, err
return partialResult(), err
}
llmDecisions := make([]Decision, 0)
for _, batch := range batches {
if err := ctx.Err(); err != nil {
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
return partialResult(), err
}
transcriptDescription := ""
if req.Config != nil {
@@ -129,7 +135,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
if err != nil {
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
return partialResult(), err
}
var response LLMValidationResponse
@@ -191,7 +197,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...)
continue
}
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
}
reconciled := reconcileLLMResponse(batch.Items, response)
@@ -201,12 +207,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
operation.ProposalIndices = reconciled.UnavailableIndices
} else {
operation.Succeeded = true
if completion.Execution.StructuralRepairs > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
}
if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
}
operation.RecoveryReasons = outcomes.ExecutionRecoveries(completion.Execution)
if len(reconciled.IgnoredIndices) > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryValidationDecisionsFiltered)
}
@@ -218,14 +219,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
llmDecisions = append(llmDecisions, reconciled.Decisions...)
}
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
all = append(all, llmDecisions...)
result := partialResult()
all := result.Decisions
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
return result, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
return result, nil
}
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
@@ -251,28 +251,31 @@ func errPayload(err error) any {
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
immediate := make([]Decision, 0)
segments := make(map[int]schema.Segment)
var segments []schema.Segment
positions := make(map[int]int)
vocabulary := NewProtectedVocabulary(req.Glossary)
if req.WorkingTranscript != nil {
segments = make(map[int]schema.Segment, len(req.WorkingTranscript.Segments))
for _, seg := range req.WorkingTranscript.Segments {
segments[seg.ID] = seg
segments = req.WorkingTranscript.Segments
for i, seg := range segments {
positions[seg.ID] = i
}
}
for _, p := range req.CandidateProposal {
seg, ok := segments[p.TargetSegmentID]
position, ok := positions[p.TargetSegmentID]
if !ok {
immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found"))
continue
}
seg := segments[position]
preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy)
if !preview.Applicable {
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
continue
}
neighbors, glossaryEntries := validationContext(req, seg.ID, preview.CorrectedSegmentText)
neighbors, glossaryEntries := validationContext(segments, position, req.Glossary, vocabulary, preview.CorrectedSegmentText)
items = append(items, LLMValidationItem{
CorrectionIndex: p.ProposalIndex,
SegmentID: p.TargetSegmentID,