eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Simplify references and output instructions in LLM prompts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 12:38:23 +00:00
parent 325c4ea7d3
commit 011b021d47
31 changed files with 248 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/framework/validators/llm_batching.go
View File

@@ -70,7 +70,7 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est
}
func estimateBatchTokens(estimator chunking.TokenEstimator, batch []LLMValidationItem) (int, error) {
payload, err := json.Marshal(batch)
payload, err := json.Marshal(localValidationItems(batch))
if err != nil {
return 0, fmt.Errorf("marshal batch payload: %w", err)
}

12
internal/framework/validators/llm_context.go
View File

@@ -8,8 +8,8 @@ import (
// validationContext supplies adjacent source segments within the current chunk
// and glossary definitions mentioned in the candidate or that local context.
// These fields travel in each item so batching accounts for their size.
func validationContext(req Request, segmentID int, corrected string) ([]schema.Segment, []schema.GlossaryEntry) {
var neighbors []schema.Segment
func validationContext(req Request, segmentID int, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
var neighbors []NeighboringSegment
var source string
if req.WorkingTranscript != nil {
segments := req.WorkingTranscript.Segments
@@ -19,10 +19,10 @@ func validationContext(req Request, segmentID int, corrected string) ([]schema.S
}
source = segment.Text
if i > 0 {
neighbors = append(neighbors, segments[i-1])
neighbors = append(neighbors, neighboringSegment("preceding", segments[i-1]))
}
if i+1 < len(segments) {
neighbors = append(neighbors, segments[i+1])
neighbors = append(neighbors, neighboringSegment("following", segments[i+1]))
}
break
}
@@ -66,3 +66,7 @@ func fitValidationContext(item LLMValidationItem, budget int, estimator chunking
}
}
}
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {
return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories}
}

7
internal/framework/validators/llm_context_test.go
View File

@@ -29,7 +29,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
item := payload.Items[0]
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].ID != 1 || item.NeighboringSegments[1].ID != 3 || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
t.Fatalf("context: %+v", item)
}
variables, err := BuildValidationVariables(payload.Items, "")
@@ -40,6 +40,9 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
t.Fatal(err)
}
if strings.Contains(variables["PayloadJSON"], `"id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
}
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Summary != "A traveling troupe" {
t.Fatalf("rendered context: %+v", rendered)
}
@@ -60,7 +63,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
t.Fatal(err)
}
enriched := base
enriched.NeighboringSegments = []schema.Segment{{ID: 2, Text: strings.Repeat("long context ", 300)}}
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []schema.GlossaryEntry{{Name: "cat", Summary: strings.Repeat("long definition ", 300)}}
fitted := fitValidationContext(enriched, required, estimator)
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})

2
internal/framework/validators/llm_integration_test.go
View File

@@ -18,7 +18,7 @@ import (
)
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":0,"approved":true,"confidence":0.9,"reason":"ok"}]}`
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
for _, tc := range []struct {
name string
responses []string

16
internal/framework/validators/llm_models.go
View File

@@ -17,16 +17,17 @@ type StructuredCompletionRequest = structuredoutput.Request
type StructuredCompletionResponse = structuredoutput.Response
type StructuredLLMClient = structuredoutput.Client
// LLMValidationItem is one candidate correction payload passed to LLM validators.
// LLMValidationItem retains the internal proposal index until prompt encoding.
// BuildValidationVariables substitutes a one-based reference for each batch.
type LLMValidationItem struct {
CorrectionIndex int `json:"correction_index"`
SegmentID int `json:"id"`
SegmentID int `json:"-"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
OriginalSegmentText string `json:"original_segment_text"`
CorrectedSegmentText string `json:"corrected_segment_text"`
Categories []string `json:"categories,omitempty"`
NeighboringSegments []schema.Segment `json:"neighboring_segments,omitempty"`
NeighboringSegments []NeighboringSegment `json:"neighboring_segments,omitempty"`
GlossaryEntries []schema.GlossaryEntry `json:"glossary_entries,omitempty"`
}
@@ -41,6 +42,7 @@ type LLMValidationRequest struct {
Items []LLMValidationItem `json:"corrections"`
}
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
type LLMValidationDecision struct {
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
@@ -51,3 +53,11 @@ type LLMValidationDecision struct {
type LLMValidationResponse struct {
Validations []LLMValidationDecision `json:"validations"`
}
// NeighboringSegment carries semantic context without internal transcript identity.
type NeighboringSegment struct {
Position string `json:"position"`
Speaker string `json:"speaker"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}

27
internal/framework/validators/llm_partial_test.go
View File

@@ -2,6 +2,7 @@ package validators
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
@@ -47,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
@@ -72,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
@@ -84,6 +85,18 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 {
t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err)
}
for _, call := range append(client.calls, nextClient.calls...) {
var items []LLMValidationItem
if err := json.Unmarshal([]byte(call.Variables["PayloadJSON"]), &items); err != nil {
t.Fatal(err)
}
if len(items) != 1 || items[0].CorrectionIndex != 1 {
t.Fatalf("batch references must restart: %+v", items)
}
}
if !next.Decisions[0].Approved || !next.Decisions[1].Approved || next.Decisions[0].ProposalIndex != 0 || next.Decisions[1].ProposalIndex != 2 {
t.Fatalf("rebatching lost identities: %+v", next.Decisions)
}
}
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
@@ -103,7 +116,7 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
@@ -124,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})

11
internal/framework/validators/llm_prompt_variables.go
View File

@@ -8,7 +8,7 @@ import (
)
func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptDescription string) (map[string]string, error) {
payloadJSON, err := json.MarshalIndent(validationPayload, "", " ")
payloadJSON, err := json.MarshalIndent(localValidationItems(validationPayload), "", " ")
if err != nil {
return nil, fmt.Errorf("marshal validation payload: %w", err)
}
@@ -17,3 +17,12 @@ func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptD
"PayloadJSON": string(payloadJSON),
}, nil
}
// localValidationItems gives each actual batch its own short references.
func localValidationItems(items []LLMValidationItem) []LLMValidationItem {
local := append([]LLMValidationItem(nil), items...)
for i := range local {
local[i].CorrectionIndex = i + 1
}
return local
}

21
internal/framework/validators/llm_validators.go
View File

@@ -160,11 +160,12 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items),
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
},
completion.Prepared,
responsePayload,
@@ -303,22 +304,18 @@ type validationReconciliation struct {
}
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
expected := make(map[int]struct{}, len(items))
for _, item := range items {
expected[item.CorrectionIndex] = struct{}{}
}
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
seen := make(map[int][]LLMValidationDecision, len(items))
for _, d := range response.Validations {
if _, ok := expected[d.CorrectionIndex]; !ok {
if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) {
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
continue
}
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
}
for _, item := range items {
entries := seen[item.CorrectionIndex]
for i, item := range items {
entries := seen[i+1]
if len(entries) == 0 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)

23
internal/framework/validators/llm_validators_test.go
View File

@@ -152,8 +152,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
}}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
@@ -224,13 +224,14 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
unavailable []int
ignored []int
}{
{"missing and extra", []LLMValidationDecision{approve(7), reject(18), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"conflicting duplicate", []LLMValidationDecision{approve(7), approve(11), reject(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(7), approve(11), approve(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(18), approve(7), approve(99), approve(11)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(18), approve(11), approve(7)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(7), {CorrectionIndex: 11, Approved: true, Confidence: 1.2}, reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
} {
t.Run(tc.name, func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
@@ -337,7 +338,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
@@ -376,7 +377,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}}
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil {
t.Fatal(err)
@@ -401,7 +402,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }