eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Simplify references and output instructions in LLM prompts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 12:38:23 +00:00
parent 325c4ea7d3
commit 011b021d47
31 changed files with 248 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/framework/promptcontext/transcript_section.go
View File

@@ -21,7 +21,8 @@ type transcriptSectionPayload struct {
}
// MarshalTranscriptSectionJSON builds the standardized transcript-section JSON
// payload consumed by proposal prompt templates.
// payload consumed by proposal prompt templates. IDs are one-based positions in
// this request; the transcript and its actual IDs are unchanged.
func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex int) ([]byte, error) {
payload := transcriptSectionPayload{
SectionIndex: sectionIndex,
@@ -29,9 +30,9 @@ func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex in
}
if transcript != nil {
for _, s := range transcript.Segments {
for i, s := range transcript.Segments {
payload.Segments = append(payload.Segments, transcriptSectionSegment{
ID: s.ID,
ID: i + 1,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,

2
internal/framework/promptcontext/transcript_section_test.go
View File

@@ -10,7 +10,7 @@ import (
func TestMarshalTranscriptSectionJSONShape(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}},
{ID: 817, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}},
}}
raw, err := MarshalTranscriptSectionJSON(transcript, 3)

2
internal/framework/proposal_generation/adapter_integration_test.go
View File

@@ -61,7 +61,7 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
generated, err := GenerateCandidates(context.Background(), Request{ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
if err != nil {
t.Fatal(err)
}

28
internal/framework/proposal_generation/generate.go
View File

@@ -36,10 +36,10 @@ type InteractionArtifacts struct {
// StructuredCorrectionProposal is one LLM response correction payload.
type StructuredCorrectionProposal struct {
TargetSegmentID int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
SegmentReference int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
}
// StructuredCorrectionSet is the reusable structured LLM response model for
@@ -142,6 +142,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"segment_ids": segmentIDs(req.WorkingTranscript),
"prompt": completion.Prompt,
}
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
@@ -185,8 +186,13 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
for i, raw := range response.Corrections {
// Unknown references remain invalid candidates for the existing domain validators.
targetID := 0
if req.WorkingTranscript != nil && raw.SegmentReference > 0 && raw.SegmentReference <= len(req.WorkingTranscript.Segments) {
targetID = req.WorkingTranscript.Segments[raw.SegmentReference-1].ID
}
candidate := proposals.CorrectionProposal{
TargetSegmentID: raw.TargetSegmentID,
TargetSegmentID: targetID,
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
Confidence: raw.Confidence,
@@ -247,3 +253,15 @@ func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata
ErrorPayloadPath: artifacts.ErrorPayloadPath,
}, nil
}
// segmentIDs records the request-local reference mapping in debug metadata.
func segmentIDs(transcript *schema.Transcript) []int {
if transcript == nil {
return nil
}
ids := make([]int, len(transcript.Segments))
for i, segment := range transcript.Segments {
ids[i] = segment.ID
}
return ids
}

28
internal/framework/proposal_generation/generate_test.go
View File

@@ -12,6 +12,7 @@ import (
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
@@ -77,6 +78,7 @@ func defaultRequest(t *testing.T) Request {
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
StartIndex: 0,
}
}
@@ -86,7 +88,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 7, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
},
},
},
@@ -104,6 +106,9 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
t.Fatalf("unexpected proposal lengths: %+v", got)
}
if got.Corrections[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Corrections)
}
if got.Enriched[0].ProposalIndex != 10 {
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
}
@@ -118,7 +123,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
req := defaultRequest(t)
@@ -140,7 +145,7 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
diag := &captureDiagnosticsWriter{}
@@ -172,7 +177,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
{SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
},
},
},
@@ -235,8 +240,8 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
{TargetSegmentID: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
},
}
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
@@ -266,8 +271,8 @@ func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 3, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
},
}
section0 := contracts.SectionMetadata{Index: 0}
@@ -276,6 +281,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
req0 := defaultRequest(t)
req0.LLMClient = client
req0.Section = &section0
req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}}
req0.StartIndex = 0
part0, err := GenerateCandidates(context.Background(), req0)
if err != nil {
@@ -285,6 +291,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
req1 := defaultRequest(t)
req1.LLMClient = client
req1.Section = &section1
req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}}
req1.StartIndex = len(part0.Enriched)
part1, err := GenerateCandidates(context.Background(), req1)
if err != nil {
@@ -295,6 +302,9 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
if len(all) != 2 {
t.Fatalf("expected 2 proposals, got %d", len(all))
}
if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 {
t.Fatalf("section references did not reset: %+v", all)
}
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
}
@@ -308,7 +318,7 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "proposal-validation-secret"
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
},
}
t.Setenv("PRIMARY_SECRET", primarySecret)

47
internal/framework/proposal_generation/module_proposal_test.go
View File

@@ -2,6 +2,7 @@ package proposal_generation
import (
"context"
"encoding/json"
"errors"
"testing"
@@ -17,7 +18,7 @@ import (
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
section := contracts.SectionMetadata{Index: 7}
@@ -135,3 +136,47 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
t.Fatalf("fatal operation lost: %+v, %v", result, err)
}
}
func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}}
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9},
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
}}}}
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{WorkingTranscript: transcript},
RunSpec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique},
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal, BuildVariables: BuildProposalVariables,
})
if err != nil {
t.Fatal(err)
}
var payload struct {
Segments []struct {
ID int `json:"id"`
Text string `json:"text"`
} `json:"segments"`
}
if err := json.Unmarshal([]byte(client.calls[0].Variables["SectionJSON"]), &payload); err != nil {
t.Fatal(err)
}
if len(payload.Segments) != 2 || payload.Segments[0].ID != 1 || payload.Segments[1].ID != 2 || payload.Segments[1].Text != "uh hello" {
t.Fatalf("request references: %+v", payload)
}
if len(result.Proposals) != 4 {
t.Fatalf("proposals: %+v", result)
}
for i, id := range []int{903, 71, 903, 0} {
if result.Proposals[i].TargetSegmentID != id {
t.Fatalf("mapped proposal %d: %+v", i, result.Proposals[i])
}
}
if result.Proposals[0].CorrectedText != "" || transcript.Segments[0].ID != 71 || transcript.Segments[1].ID != 903 {
t.Fatal("mapping changed deletion or source identities")
}
}

26
internal/framework/runner/runner_test.go
View File

@@ -621,7 +621,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
},
}
}
@@ -653,7 +653,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
target := out.(*validators.LLMValidationResponse)
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: section, Approved: true, Confidence: 0.99, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"},
},
}
return contracts.StructuredCompletionResponse{}, nil
@@ -942,7 +942,7 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
}
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}}}
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("NewLLMBackedValidator: %v", err)
@@ -969,7 +969,7 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.95, Reason: "reject"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "reject"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
@@ -1028,8 +1028,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 3, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"},
}}}}
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
@@ -1095,8 +1095,8 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Confidence: 0.9, Reason: "dup"},
}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1122,7 +1122,7 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
func TestRunnerLLMValidatorBatching(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
@@ -1159,7 +1159,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
primarySecret := "runner-primary-secret"
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
t.Setenv("RUNNER_PRIMARY_KEY", primarySecret)
t.Setenv("RUNNER_VALIDATION_KEY", validationSecret)
@@ -1265,7 +1265,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
},
},
},
@@ -1308,7 +1308,7 @@ func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5},
{SegmentReference: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5},
},
},
},
@@ -1342,7 +1342,7 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5},
{SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5},
},
},
},

2
internal/framework/validators/llm_batching.go
View File

@@ -70,7 +70,7 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est
}
func estimateBatchTokens(estimator chunking.TokenEstimator, batch []LLMValidationItem) (int, error) {
payload, err := json.Marshal(batch)
payload, err := json.Marshal(localValidationItems(batch))
if err != nil {
return 0, fmt.Errorf("marshal batch payload: %w", err)
}

12
internal/framework/validators/llm_context.go
View File

@@ -8,8 +8,8 @@ import (
// validationContext supplies adjacent source segments within the current chunk
// and glossary definitions mentioned in the candidate or that local context.
// These fields travel in each item so batching accounts for their size.
func validationContext(req Request, segmentID int, corrected string) ([]schema.Segment, []schema.GlossaryEntry) {
var neighbors []schema.Segment
func validationContext(req Request, segmentID int, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
var neighbors []NeighboringSegment
var source string
if req.WorkingTranscript != nil {
segments := req.WorkingTranscript.Segments
@@ -19,10 +19,10 @@ func validationContext(req Request, segmentID int, corrected string) ([]schema.S
}
source = segment.Text
if i > 0 {
neighbors = append(neighbors, segments[i-1])
neighbors = append(neighbors, neighboringSegment("preceding", segments[i-1]))
}
if i+1 < len(segments) {
neighbors = append(neighbors, segments[i+1])
neighbors = append(neighbors, neighboringSegment("following", segments[i+1]))
}
break
}
@@ -66,3 +66,7 @@ func fitValidationContext(item LLMValidationItem, budget int, estimator chunking
}
}
}
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {
return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories}
}

7
internal/framework/validators/llm_context_test.go
View File

@@ -29,7 +29,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
item := payload.Items[0]
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].ID != 1 || item.NeighboringSegments[1].ID != 3 || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
t.Fatalf("context: %+v", item)
}
variables, err := BuildValidationVariables(payload.Items, "")
@@ -40,6 +40,9 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
t.Fatal(err)
}
if strings.Contains(variables["PayloadJSON"], `"id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
}
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Summary != "A traveling troupe" {
t.Fatalf("rendered context: %+v", rendered)
}
@@ -60,7 +63,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
t.Fatal(err)
}
enriched := base
enriched.NeighboringSegments = []schema.Segment{{ID: 2, Text: strings.Repeat("long context ", 300)}}
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []schema.GlossaryEntry{{Name: "cat", Summary: strings.Repeat("long definition ", 300)}}
fitted := fitValidationContext(enriched, required, estimator)
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})

2
internal/framework/validators/llm_integration_test.go
View File

@@ -18,7 +18,7 @@ import (
)
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":0,"approved":true,"confidence":0.9,"reason":"ok"}]}`
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
for _, tc := range []struct {
name string
responses []string

16
internal/framework/validators/llm_models.go
View File

@@ -17,16 +17,17 @@ type StructuredCompletionRequest = structuredoutput.Request
type StructuredCompletionResponse = structuredoutput.Response
type StructuredLLMClient = structuredoutput.Client
// LLMValidationItem is one candidate correction payload passed to LLM validators.
// LLMValidationItem retains the internal proposal index until prompt encoding.
// BuildValidationVariables substitutes a one-based reference for each batch.
type LLMValidationItem struct {
CorrectionIndex int `json:"correction_index"`
SegmentID int `json:"id"`
SegmentID int `json:"-"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
OriginalSegmentText string `json:"original_segment_text"`
CorrectedSegmentText string `json:"corrected_segment_text"`
Categories []string `json:"categories,omitempty"`
NeighboringSegments []schema.Segment `json:"neighboring_segments,omitempty"`
NeighboringSegments []NeighboringSegment `json:"neighboring_segments,omitempty"`
GlossaryEntries []schema.GlossaryEntry `json:"glossary_entries,omitempty"`
}
@@ -41,6 +42,7 @@ type LLMValidationRequest struct {
Items []LLMValidationItem `json:"corrections"`
}
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
type LLMValidationDecision struct {
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
@@ -51,3 +53,11 @@ type LLMValidationDecision struct {
type LLMValidationResponse struct {
Validations []LLMValidationDecision `json:"validations"`
}
// NeighboringSegment carries semantic context without internal transcript identity.
type NeighboringSegment struct {
Position string `json:"position"`
Speaker string `json:"speaker"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}

27
internal/framework/validators/llm_partial_test.go
View File

@@ -2,6 +2,7 @@ package validators
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
@@ -47,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
@@ -72,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
@@ -84,6 +85,18 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 {
t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err)
}
for _, call := range append(client.calls, nextClient.calls...) {
var items []LLMValidationItem
if err := json.Unmarshal([]byte(call.Variables["PayloadJSON"]), &items); err != nil {
t.Fatal(err)
}
if len(items) != 1 || items[0].CorrectionIndex != 1 {
t.Fatalf("batch references must restart: %+v", items)
}
}
if !next.Decisions[0].Approved || !next.Decisions[1].Approved || next.Decisions[0].ProposalIndex != 0 || next.Decisions[1].ProposalIndex != 2 {
t.Fatalf("rebatching lost identities: %+v", next.Decisions)
}
}
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
@@ -103,7 +116,7 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
@@ -124,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})

11
internal/framework/validators/llm_prompt_variables.go
View File

@@ -8,7 +8,7 @@ import (
)
func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptDescription string) (map[string]string, error) {
payloadJSON, err := json.MarshalIndent(validationPayload, "", " ")
payloadJSON, err := json.MarshalIndent(localValidationItems(validationPayload), "", " ")
if err != nil {
return nil, fmt.Errorf("marshal validation payload: %w", err)
}
@@ -17,3 +17,12 @@ func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptD
"PayloadJSON": string(payloadJSON),
}, nil
}
// localValidationItems gives each actual batch its own short references.
func localValidationItems(items []LLMValidationItem) []LLMValidationItem {
local := append([]LLMValidationItem(nil), items...)
for i := range local {
local[i].CorrectionIndex = i + 1
}
return local
}

21
internal/framework/validators/llm_validators.go
View File

@@ -160,11 +160,12 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items),
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
},
completion.Prepared,
responsePayload,
@@ -303,22 +304,18 @@ type validationReconciliation struct {
}
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
expected := make(map[int]struct{}, len(items))
for _, item := range items {
expected[item.CorrectionIndex] = struct{}{}
}
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
seen := make(map[int][]LLMValidationDecision, len(items))
for _, d := range response.Validations {
if _, ok := expected[d.CorrectionIndex]; !ok {
if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) {
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
continue
}
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
}
for _, item := range items {
entries := seen[item.CorrectionIndex]
for i, item := range items {
entries := seen[i+1]
if len(entries) == 0 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)

23
internal/framework/validators/llm_validators_test.go
View File

@@ -152,8 +152,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
}}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
@@ -224,13 +224,14 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
unavailable []int
ignored []int
}{
{"missing and extra", []LLMValidationDecision{approve(7), reject(18), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"conflicting duplicate", []LLMValidationDecision{approve(7), approve(11), reject(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(7), approve(11), approve(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(18), approve(7), approve(99), approve(11)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(18), approve(11), approve(7)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(7), {CorrectionIndex: 11, Approved: true, Confidence: 1.2}, reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
} {
t.Run(tc.name, func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
@@ -337,7 +338,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
@@ -376,7 +377,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}}
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil {
t.Fatal(err)
@@ -401,7 +402,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }