Simplify references and output instructions in LLM prompts

This commit is contained in:
2026-09-25 12:38:23 +00:00
parent 325c4ea7d3
commit 011b021d47
31 changed files with 248 additions and 129 deletions

View File

@@ -143,12 +143,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
if validation {
if p.mode == "span-clean" || p.mode == "span-filter" {
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":2,"approved":true,"confidence":0.99,"reason":"ok"}]}`
} else {
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
}
if p.mode == "span-negative" && validationCall == 2 {
content = `{"validations":[{"correction_index":0,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}`
content = `{"validations":[{"correction_index":1,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}`
}
}
}
@@ -159,10 +159,8 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
}
if validation && !strings.HasPrefix(p.mode, "span-") {
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
if p.mode == "repair-filter-later-failure" {
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
} else if p.mode == "invalid-decision-after-repair" {
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
if p.mode == "invalid-decision-after-repair" {
if validationCall == 1 {
content = `{"wrong":true}`
} else {
@@ -173,12 +171,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
if validation && strings.HasPrefix(p.mode, "decision-") {
decisions := make([]map[string]any, 0, len(requestedIndices)+2)
for _, index := range requestedIndices {
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2 {
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 {
continue
}
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 1)
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2)
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"})
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 {
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 4 {
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"})
}
}
@@ -606,9 +604,9 @@ func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse(t *testing.T) {
full, survivor := 0, 0
for _, indices := range requests {
switch {
case reflect.DeepEqual(indices, []int{0, 1, 2, 3}):
case reflect.DeepEqual(indices, []int{1, 2, 3, 4}):
full++
case reflect.DeepEqual(indices, []int{0}):
case reflect.DeepEqual(indices, []int{1}):
survivor++
default:
t.Fatalf("unexpected validation identities: %v", requests)
@@ -662,7 +660,7 @@ func TestSubprocessReportsFilteredExtraValidatorDecision(t *testing.T) {
}
requests := f.assertDecisionRequestsUseOneAttempt(t)
for _, indices := range requests {
if !reflect.DeepEqual(indices, []int{0}) {
if !reflect.DeepEqual(indices, []int{1}) {
t.Fatalf("unexpected identities: %v", requests)
}
}

View File

@@ -42,7 +42,13 @@ transcript section, and trimmed description once for all four modules. PromptKit
renders their embedded definitions from those variables and validates the
correction-set schema before Audita maps proposals. Domain validators still
check confidence, applicability, protected glossary terms, and replacement
policy. Shared context encoding lives in `internal/framework/promptcontext`.
policy. Model-facing segment IDs are one-based positions within the proposal
request. Generation maps returned references back to the current chunk's actual
segment IDs, preserving response order and multiple proposals per segment.
Unknown references become invalid-target candidates for the existing domain
checks; they are never interpreted as actual transcript IDs. Debug request
metadata records `segment_ids` in local-reference order (reference 1 is the
first entry). Shared context encoding lives in `internal/framework/promptcontext`.
An empty `corrected_text` deletes the identified span when the resulting segment
remains nonblank. Whitespace-only original spans can express spacing corrections;
matching stays literal under each module's replacement policy. No automatic

View File

@@ -42,9 +42,14 @@ pass these values through the role-bound adapter. See the
Proposal instructions describe literal matching, each module's replacement policy,
source-based proposal identity, and confidence in whether an edit is warranted.
Validator instructions distinguish correction indices from segment IDs and confidence
in the verdict from the verdict itself. The grammar article exception and glossary
mishearing exception apply consistently across their instructions.
Proposal `id` and validator `correction_index` values are short, one-based
references scoped to each request, not internal transcript/proposal identities.
Mapping belongs to [proposal generation](modules.md) and
[validator reconciliation](validators.md). Validator instructions distinguish
confidence in the verdict from the verdict itself. Compact output examples
illustrate the supplied schema without requiring fixed JSON formatting or key
order. The grammar article exception and glossary mishearing exception apply
consistently across their instructions.
Only the three active validator definitions are retained. The unused internal
`grammar_review` and `spoken_word_review` definitions and type selections were

View File

@@ -63,12 +63,14 @@ Oversized single inputs are rejected with a consolidated
model failures make only their batch candidates unavailable while other
batches continue. PromptKit validates each complete response against the
validator-decision JSON Schema and performs configured structural repair before
Audita receives decoded decisions. Audita then reconciles those decisions with
the correction indices requested in that batch: one usable entry keeps its
Audita receives decoded decisions. Each actual batch presents candidates
numbered from 1, independent of their internal proposal indices. Audita then
reconciles those local references: one usable entry keeps its
approval or editorial rejection; a missing or duplicate requested entry makes
only that candidate unavailable; unexpected indices are ignored. An empty list
makes all requested candidates unavailable. Results are ordered by request
identity and satisfy strict internal cardinality before the runner advances
makes all requested candidates unavailable. Decisions and unavailable indices
map back to internal proposal identities; results are ordered by request identity
and satisfy strict internal cardinality before the runner advances
approved candidates. An unavailable candidate never advances as approved.
Invalid confidence in a decoded test double also makes only its candidate
unavailable; production schema validation still rejects out-of-range confidence
@@ -79,6 +81,12 @@ chain without returning unchecked candidates as approved. Reconciliation makes
no semantic retry or extra model call; guided recovery remains
[deferred](../roadmap/future.md#domain-aware-response-recovery).
Debug request metadata records `proposal_indices` in local-reference order
(reference 1 is the first entry). Raw decoded responses and ignored indices use
local references; reconciled decisions and unavailable indices use internal
proposal indices. Batch token estimates use the same locally numbered payload
as prompt encoding.
## Decision and Rejection Reporting
Runner collects decisions, rejected candidates, and operation outcomes. Ledger
@@ -111,9 +119,11 @@ is covered by [report assembly](diagnostics-reporting.md).
## Supplemental Validation Context
Each candidate includes the immediately preceding and following source segments
available within its fixed chunk, plus glossary entries whose names, aliases, or
available within its fixed chunk, labeled `preceding` or `following` with speaker,
text, and categories, plus glossary entries whose names, aliases, or
recognized plurals appear in its original/resulting text or those neighbors.
Glossary matching reuses the protected vocabulary; entries retain glossary order.
Model-facing candidate and neighboring context omit internal segment IDs.
This supplies evidence such as a troupe definition for a proposed `Jesters`
correction or an adjacent speaker turn explaining deliberate repetition. A correction
already clear from its own segment does not require such evidence. No audio,

View File

@@ -21,7 +21,8 @@ type transcriptSectionPayload struct {
}
// MarshalTranscriptSectionJSON builds the standardized transcript-section JSON
// payload consumed by proposal prompt templates.
// payload consumed by proposal prompt templates. IDs are one-based positions in
// this request; the transcript and its actual IDs are unchanged.
func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex int) ([]byte, error) {
payload := transcriptSectionPayload{
SectionIndex: sectionIndex,
@@ -29,9 +30,9 @@ func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex in
}
if transcript != nil {
for _, s := range transcript.Segments {
for i, s := range transcript.Segments {
payload.Segments = append(payload.Segments, transcriptSectionSegment{
ID: s.ID,
ID: i + 1,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,

View File

@@ -10,7 +10,7 @@ import (
func TestMarshalTranscriptSectionJSONShape(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}},
{ID: 817, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}},
}}
raw, err := MarshalTranscriptSectionJSON(transcript, 3)

View File

@@ -61,7 +61,7 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
generated, err := GenerateCandidates(context.Background(), Request{ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
if err != nil {
t.Fatal(err)
}

View File

@@ -36,10 +36,10 @@ type InteractionArtifacts struct {
// StructuredCorrectionProposal is one LLM response correction payload.
type StructuredCorrectionProposal struct {
TargetSegmentID int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
SegmentReference int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
}
// StructuredCorrectionSet is the reusable structured LLM response model for
@@ -142,6 +142,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"segment_ids": segmentIDs(req.WorkingTranscript),
"prompt": completion.Prompt,
}
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
@@ -185,8 +186,13 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
for i, raw := range response.Corrections {
// Unknown references remain invalid candidates for the existing domain validators.
targetID := 0
if req.WorkingTranscript != nil && raw.SegmentReference > 0 && raw.SegmentReference <= len(req.WorkingTranscript.Segments) {
targetID = req.WorkingTranscript.Segments[raw.SegmentReference-1].ID
}
candidate := proposals.CorrectionProposal{
TargetSegmentID: raw.TargetSegmentID,
TargetSegmentID: targetID,
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
Confidence: raw.Confidence,
@@ -247,3 +253,15 @@ func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata
ErrorPayloadPath: artifacts.ErrorPayloadPath,
}, nil
}
// segmentIDs records the request-local reference mapping in debug metadata.
func segmentIDs(transcript *schema.Transcript) []int {
if transcript == nil {
return nil
}
ids := make([]int, len(transcript.Segments))
for i, segment := range transcript.Segments {
ids[i] = segment.ID
}
return ids
}

View File

@@ -12,6 +12,7 @@ import (
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
@@ -77,6 +78,7 @@ func defaultRequest(t *testing.T) Request {
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
StartIndex: 0,
}
}
@@ -86,7 +88,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 7, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
},
},
},
@@ -104,6 +106,9 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
t.Fatalf("unexpected proposal lengths: %+v", got)
}
if got.Corrections[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Corrections)
}
if got.Enriched[0].ProposalIndex != 10 {
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
}
@@ -118,7 +123,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
req := defaultRequest(t)
@@ -140,7 +145,7 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
diag := &captureDiagnosticsWriter{}
@@ -172,7 +177,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
{SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
},
},
},
@@ -235,8 +240,8 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
{TargetSegmentID: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
},
}
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
@@ -266,8 +271,8 @@ func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 3, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
},
}
section0 := contracts.SectionMetadata{Index: 0}
@@ -276,6 +281,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
req0 := defaultRequest(t)
req0.LLMClient = client
req0.Section = &section0
req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}}
req0.StartIndex = 0
part0, err := GenerateCandidates(context.Background(), req0)
if err != nil {
@@ -285,6 +291,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
req1 := defaultRequest(t)
req1.LLMClient = client
req1.Section = &section1
req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}}
req1.StartIndex = len(part0.Enriched)
part1, err := GenerateCandidates(context.Background(), req1)
if err != nil {
@@ -295,6 +302,9 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
if len(all) != 2 {
t.Fatalf("expected 2 proposals, got %d", len(all))
}
if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 {
t.Fatalf("section references did not reset: %+v", all)
}
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
}
@@ -308,7 +318,7 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "proposal-validation-secret"
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
},
}
t.Setenv("PRIMARY_SECRET", primarySecret)

View File

@@ -2,6 +2,7 @@ package proposal_generation
import (
"context"
"encoding/json"
"errors"
"testing"
@@ -17,7 +18,7 @@ import (
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
section := contracts.SectionMetadata{Index: 7}
@@ -135,3 +136,47 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
t.Fatalf("fatal operation lost: %+v, %v", result, err)
}
}
func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}}
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9},
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
}}}}
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{WorkingTranscript: transcript},
RunSpec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique},
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal, BuildVariables: BuildProposalVariables,
})
if err != nil {
t.Fatal(err)
}
var payload struct {
Segments []struct {
ID int `json:"id"`
Text string `json:"text"`
} `json:"segments"`
}
if err := json.Unmarshal([]byte(client.calls[0].Variables["SectionJSON"]), &payload); err != nil {
t.Fatal(err)
}
if len(payload.Segments) != 2 || payload.Segments[0].ID != 1 || payload.Segments[1].ID != 2 || payload.Segments[1].Text != "uh hello" {
t.Fatalf("request references: %+v", payload)
}
if len(result.Proposals) != 4 {
t.Fatalf("proposals: %+v", result)
}
for i, id := range []int{903, 71, 903, 0} {
if result.Proposals[i].TargetSegmentID != id {
t.Fatalf("mapped proposal %d: %+v", i, result.Proposals[i])
}
}
if result.Proposals[0].CorrectedText != "" || transcript.Segments[0].ID != 71 || transcript.Segments[1].ID != 903 {
t.Fatal("mapping changed deletion or source identities")
}
}

View File

@@ -621,7 +621,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
},
}
}
@@ -653,7 +653,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
target := out.(*validators.LLMValidationResponse)
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: section, Approved: true, Confidence: 0.99, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"},
},
}
return contracts.StructuredCompletionResponse{}, nil
@@ -942,7 +942,7 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
}
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}}}
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("NewLLMBackedValidator: %v", err)
@@ -969,7 +969,7 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.95, Reason: "reject"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "reject"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
@@ -1028,8 +1028,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 3, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"},
}}}}
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
@@ -1095,8 +1095,8 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Confidence: 0.9, Reason: "dup"},
}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1122,7 +1122,7 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
func TestRunnerLLMValidatorBatching(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
@@ -1159,7 +1159,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
primarySecret := "runner-primary-secret"
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
t.Setenv("RUNNER_PRIMARY_KEY", primarySecret)
t.Setenv("RUNNER_VALIDATION_KEY", validationSecret)
@@ -1265,7 +1265,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
},
},
},
@@ -1308,7 +1308,7 @@ func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5},
{SegmentReference: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5},
},
},
},
@@ -1342,7 +1342,7 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5},
{SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5},
},
},
},

View File

@@ -70,7 +70,7 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est
}
func estimateBatchTokens(estimator chunking.TokenEstimator, batch []LLMValidationItem) (int, error) {
payload, err := json.Marshal(batch)
payload, err := json.Marshal(localValidationItems(batch))
if err != nil {
return 0, fmt.Errorf("marshal batch payload: %w", err)
}

View File

@@ -8,8 +8,8 @@ import (
// validationContext supplies adjacent source segments within the current chunk
// and glossary definitions mentioned in the candidate or that local context.
// These fields travel in each item so batching accounts for their size.
func validationContext(req Request, segmentID int, corrected string) ([]schema.Segment, []schema.GlossaryEntry) {
var neighbors []schema.Segment
func validationContext(req Request, segmentID int, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
var neighbors []NeighboringSegment
var source string
if req.WorkingTranscript != nil {
segments := req.WorkingTranscript.Segments
@@ -19,10 +19,10 @@ func validationContext(req Request, segmentID int, corrected string) ([]schema.S
}
source = segment.Text
if i > 0 {
neighbors = append(neighbors, segments[i-1])
neighbors = append(neighbors, neighboringSegment("preceding", segments[i-1]))
}
if i+1 < len(segments) {
neighbors = append(neighbors, segments[i+1])
neighbors = append(neighbors, neighboringSegment("following", segments[i+1]))
}
break
}
@@ -66,3 +66,7 @@ func fitValidationContext(item LLMValidationItem, budget int, estimator chunking
}
}
}
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {
return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories}
}

View File

@@ -29,7 +29,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
item := payload.Items[0]
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].ID != 1 || item.NeighboringSegments[1].ID != 3 || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
t.Fatalf("context: %+v", item)
}
variables, err := BuildValidationVariables(payload.Items, "")
@@ -40,6 +40,9 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
t.Fatal(err)
}
if strings.Contains(variables["PayloadJSON"], `"id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
}
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Summary != "A traveling troupe" {
t.Fatalf("rendered context: %+v", rendered)
}
@@ -60,7 +63,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
t.Fatal(err)
}
enriched := base
enriched.NeighboringSegments = []schema.Segment{{ID: 2, Text: strings.Repeat("long context ", 300)}}
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []schema.GlossaryEntry{{Name: "cat", Summary: strings.Repeat("long definition ", 300)}}
fitted := fitValidationContext(enriched, required, estimator)
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})

View File

@@ -18,7 +18,7 @@ import (
)
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":0,"approved":true,"confidence":0.9,"reason":"ok"}]}`
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
for _, tc := range []struct {
name string
responses []string

View File

@@ -17,16 +17,17 @@ type StructuredCompletionRequest = structuredoutput.Request
type StructuredCompletionResponse = structuredoutput.Response
type StructuredLLMClient = structuredoutput.Client
// LLMValidationItem is one candidate correction payload passed to LLM validators.
// LLMValidationItem retains the internal proposal index until prompt encoding.
// BuildValidationVariables substitutes a one-based reference for each batch.
type LLMValidationItem struct {
CorrectionIndex int `json:"correction_index"`
SegmentID int `json:"id"`
SegmentID int `json:"-"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
OriginalSegmentText string `json:"original_segment_text"`
CorrectedSegmentText string `json:"corrected_segment_text"`
Categories []string `json:"categories,omitempty"`
NeighboringSegments []schema.Segment `json:"neighboring_segments,omitempty"`
NeighboringSegments []NeighboringSegment `json:"neighboring_segments,omitempty"`
GlossaryEntries []schema.GlossaryEntry `json:"glossary_entries,omitempty"`
}
@@ -41,6 +42,7 @@ type LLMValidationRequest struct {
Items []LLMValidationItem `json:"corrections"`
}
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
type LLMValidationDecision struct {
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
@@ -51,3 +53,11 @@ type LLMValidationDecision struct {
type LLMValidationResponse struct {
Validations []LLMValidationDecision `json:"validations"`
}
// NeighboringSegment carries semantic context without internal transcript identity.
type NeighboringSegment struct {
Position string `json:"position"`
Speaker string `json:"speaker"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}

View File

@@ -2,6 +2,7 @@ package validators
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
@@ -47,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
@@ -72,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
@@ -84,6 +85,18 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 {
t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err)
}
for _, call := range append(client.calls, nextClient.calls...) {
var items []LLMValidationItem
if err := json.Unmarshal([]byte(call.Variables["PayloadJSON"]), &items); err != nil {
t.Fatal(err)
}
if len(items) != 1 || items[0].CorrectionIndex != 1 {
t.Fatalf("batch references must restart: %+v", items)
}
}
if !next.Decisions[0].Approved || !next.Decisions[1].Approved || next.Decisions[0].ProposalIndex != 0 || next.Decisions[1].ProposalIndex != 2 {
t.Fatalf("rebatching lost identities: %+v", next.Decisions)
}
}
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
@@ -103,7 +116,7 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
@@ -124,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})

View File

@@ -8,7 +8,7 @@ import (
)
func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptDescription string) (map[string]string, error) {
payloadJSON, err := json.MarshalIndent(validationPayload, "", " ")
payloadJSON, err := json.MarshalIndent(localValidationItems(validationPayload), "", " ")
if err != nil {
return nil, fmt.Errorf("marshal validation payload: %w", err)
}
@@ -17,3 +17,12 @@ func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptD
"PayloadJSON": string(payloadJSON),
}, nil
}
// localValidationItems gives each actual batch its own short references.
func localValidationItems(items []LLMValidationItem) []LLMValidationItem {
local := append([]LLMValidationItem(nil), items...)
for i := range local {
local[i].CorrectionIndex = i + 1
}
return local
}

View File

@@ -160,11 +160,12 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items),
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
},
completion.Prepared,
responsePayload,
@@ -303,22 +304,18 @@ type validationReconciliation struct {
}
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
expected := make(map[int]struct{}, len(items))
for _, item := range items {
expected[item.CorrectionIndex] = struct{}{}
}
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
seen := make(map[int][]LLMValidationDecision, len(items))
for _, d := range response.Validations {
if _, ok := expected[d.CorrectionIndex]; !ok {
if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) {
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
continue
}
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
}
for _, item := range items {
entries := seen[item.CorrectionIndex]
for i, item := range items {
entries := seen[i+1]
if len(entries) == 0 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)

View File

@@ -152,8 +152,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
}}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
@@ -224,13 +224,14 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
unavailable []int
ignored []int
}{
{"missing and extra", []LLMValidationDecision{approve(7), reject(18), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"conflicting duplicate", []LLMValidationDecision{approve(7), approve(11), reject(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(7), approve(11), approve(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(18), approve(7), approve(99), approve(11)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(18), approve(11), approve(7)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(7), {CorrectionIndex: 11, Approved: true, Confidence: 1.2}, reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
} {
t.Run(tc.name, func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
@@ -337,7 +338,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
@@ -376,7 +377,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}}
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil {
t.Fatal(err)
@@ -401,7 +402,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }

View File

@@ -93,7 +93,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94},
{SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94},
},
},
},

View File

@@ -42,10 +42,10 @@ func (f *fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.St
items, _ := payload["corrections"].([]map[string]any)
for _, item := range items {
target.Corrections = append(target.Corrections, proposal_generation.StructuredCorrectionProposal{
TargetSegmentID: item["id"].(int),
OriginalText: item["original_text"].(string),
CorrectedText: item["corrected_text"].(string),
Confidence: item["confidence"].(float64),
SegmentReference: item["id"].(int),
OriginalText: item["original_text"].(string),
CorrectedText: item["corrected_text"].(string),
Confidence: item["confidence"].(float64),
})
}
return contracts.StructuredCompletionResponse{}, nil

View File

@@ -94,7 +94,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94},
{SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94},
},
},
},

View File

@@ -94,7 +94,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "I I think", CorrectedText: "I think", Confidence: 0.94},
{SegmentReference: 1, OriginalText: "I I think", CorrectedText: "I think", Confidence: 0.94},
},
},
},

View File

@@ -19,16 +19,14 @@ messages:
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.
- Use the exact id from the input segment.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Each returned correction must contain only id, original_text, corrected_text, and confidence.
- Do not return corrections where original_text and corrected_text are identical.
- Do not return speaker, start, or end fields.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .HardeningPolicy }}

View File

@@ -20,18 +20,16 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Use the exact id from the input segment.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed.
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
- Each returned correction must contain only id, original_text, corrected_text, and confidence.
- Do not return corrections where original_text and corrected_text are identical.
- Do not return speaker, start, or end fields.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .HardeningPolicy }}

View File

@@ -19,16 +19,14 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Use the exact id from the input segment.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Each returned correction must contain only id, original_text, corrected_text, and confidence.
- Do not return corrections where original_text and corrected_text are identical.
- Do not return speaker, start, or end fields.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .HardeningPolicy }}

View File

@@ -22,18 +22,16 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Use the exact id from the input segment.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result.
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
- Each returned correction must contain only id, original_text, corrected_text, and confidence.
- Do not return corrections where original_text and corrected_text are identical.
- Do not return speaker, start, or end fields.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .HardeningPolicy }}

View File

@@ -9,7 +9,7 @@ messages:
Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.
Rules:
- Return exactly one validation decision for each supplied correction_index, including rejected corrections. Copy correction_index exactly; do not substitute the segment id, renumber entries, or invent indices. Return a JSON object with a validations array matching the supplied schema.
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
- Approve editorial revisions that preserve the underlying meaning of the segment.
- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.
- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.
@@ -21,8 +21,7 @@ messages:
- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
- An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty.
- If supplied, neighboring_segments are adjacent source segments for context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- Each returned validation must contain only correction_index, approved, confidence, and reason.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .HardeningPolicy }}

View File

@@ -9,7 +9,7 @@ messages:
Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.
Rules:
- Return exactly one validation decision for each supplied correction_index, including rejected corrections. Copy correction_index exactly; do not substitute the segment id, renumber entries, or invent indices. Return a JSON object with a validations array matching the supplied schema.
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.
- Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning.
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
@@ -17,8 +17,7 @@ messages:
- For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone.
- Do not reject a correction merely because the literal written word changes.
- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.
- If supplied, neighboring_segments are adjacent source segments for context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- Each returned validation must contain only correction_index, approved, confidence, and reason.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .HardeningPolicy }}

View File

@@ -9,15 +9,14 @@ messages:
Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.
Rules:
- Return exactly one validation decision for each supplied correction_index, including rejected corrections. Copy correction_index exactly; do not substitute the segment id, renumber entries, or invent indices. Return a JSON object with a validations array matching the supplied schema.
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
- Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.
- Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn".
- Reject unrelated substitutions like changing "Lyra" to "Jesters".
- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.
- For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription.
- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.
- If supplied, neighboring_segments are adjacent source segments for context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- Each returned validation must contain only correction_index, approved, confidence, and reason.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .HardeningPolicy }}