Simplify references and output instructions in LLM prompts

This commit is contained in:
2026-09-25 12:38:23 +00:00
parent 325c4ea7d3
commit 011b021d47
31 changed files with 248 additions and 129 deletions

View File

@@ -143,12 +143,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
} }
if validation { if validation {
if p.mode == "span-clean" || p.mode == "span-filter" { if p.mode == "span-clean" || p.mode == "span-filter" {
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}` content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":2,"approved":true,"confidence":0.99,"reason":"ok"}]}`
} else { } else {
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}` content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
} }
if p.mode == "span-negative" && validationCall == 2 { if p.mode == "span-negative" && validationCall == 2 {
content = `{"validations":[{"correction_index":0,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}` content = `{"validations":[{"correction_index":1,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}`
} }
} }
} }
@@ -159,10 +159,8 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
} }
} }
if validation && !strings.HasPrefix(p.mode, "span-") { if validation && !strings.HasPrefix(p.mode, "span-") {
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
if p.mode == "repair-filter-later-failure" {
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}` content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
} else if p.mode == "invalid-decision-after-repair" { if p.mode == "invalid-decision-after-repair" {
if validationCall == 1 { if validationCall == 1 {
content = `{"wrong":true}` content = `{"wrong":true}`
} else { } else {
@@ -173,12 +171,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
if validation && strings.HasPrefix(p.mode, "decision-") { if validation && strings.HasPrefix(p.mode, "decision-") {
decisions := make([]map[string]any, 0, len(requestedIndices)+2) decisions := make([]map[string]any, 0, len(requestedIndices)+2)
for _, index := range requestedIndices { for _, index := range requestedIndices {
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2 { if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 {
continue continue
} }
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 1) approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2)
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"}) decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"})
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 { if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 4 {
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"}) decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"})
} }
} }
@@ -606,9 +604,9 @@ func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse(t *testing.T) {
full, survivor := 0, 0 full, survivor := 0, 0
for _, indices := range requests { for _, indices := range requests {
switch { switch {
case reflect.DeepEqual(indices, []int{0, 1, 2, 3}): case reflect.DeepEqual(indices, []int{1, 2, 3, 4}):
full++ full++
case reflect.DeepEqual(indices, []int{0}): case reflect.DeepEqual(indices, []int{1}):
survivor++ survivor++
default: default:
t.Fatalf("unexpected validation identities: %v", requests) t.Fatalf("unexpected validation identities: %v", requests)
@@ -662,7 +660,7 @@ func TestSubprocessReportsFilteredExtraValidatorDecision(t *testing.T) {
} }
requests := f.assertDecisionRequestsUseOneAttempt(t) requests := f.assertDecisionRequestsUseOneAttempt(t)
for _, indices := range requests { for _, indices := range requests {
if !reflect.DeepEqual(indices, []int{0}) { if !reflect.DeepEqual(indices, []int{1}) {
t.Fatalf("unexpected identities: %v", requests) t.Fatalf("unexpected identities: %v", requests)
} }
} }

View File

@@ -42,7 +42,13 @@ transcript section, and trimmed description once for all four modules. PromptKit
renders their embedded definitions from those variables and validates the renders their embedded definitions from those variables and validates the
correction-set schema before Audita maps proposals. Domain validators still correction-set schema before Audita maps proposals. Domain validators still
check confidence, applicability, protected glossary terms, and replacement check confidence, applicability, protected glossary terms, and replacement
policy. Shared context encoding lives in `internal/framework/promptcontext`. policy. Model-facing segment IDs are one-based positions within the proposal
request. Generation maps returned references back to the current chunk's actual
segment IDs, preserving response order and multiple proposals per segment.
Unknown references become invalid-target candidates for the existing domain
checks; they are never interpreted as actual transcript IDs. Debug request
metadata records `segment_ids` in local-reference order (reference 1 is the
first entry). Shared context encoding lives in `internal/framework/promptcontext`.
An empty `corrected_text` deletes the identified span when the resulting segment An empty `corrected_text` deletes the identified span when the resulting segment
remains nonblank. Whitespace-only original spans can express spacing corrections; remains nonblank. Whitespace-only original spans can express spacing corrections;
matching stays literal under each module's replacement policy. No automatic matching stays literal under each module's replacement policy. No automatic

View File

@@ -42,9 +42,14 @@ pass these values through the role-bound adapter. See the
Proposal instructions describe literal matching, each module's replacement policy, Proposal instructions describe literal matching, each module's replacement policy,
source-based proposal identity, and confidence in whether an edit is warranted. source-based proposal identity, and confidence in whether an edit is warranted.
Validator instructions distinguish correction indices from segment IDs and confidence Proposal `id` and validator `correction_index` values are short, one-based
in the verdict from the verdict itself. The grammar article exception and glossary references scoped to each request, not internal transcript/proposal identities.
mishearing exception apply consistently across their instructions. Mapping belongs to [proposal generation](modules.md) and
[validator reconciliation](validators.md). Validator instructions distinguish
confidence in the verdict from the verdict itself. Compact output examples
illustrate the supplied schema without requiring fixed JSON formatting or key
order. The grammar article exception and glossary mishearing exception apply
consistently across their instructions.
Only the three active validator definitions are retained. The unused internal Only the three active validator definitions are retained. The unused internal
`grammar_review` and `spoken_word_review` definitions and type selections were `grammar_review` and `spoken_word_review` definitions and type selections were

View File

@@ -63,12 +63,14 @@ Oversized single inputs are rejected with a consolidated
model failures make only their batch candidates unavailable while other model failures make only their batch candidates unavailable while other
batches continue. PromptKit validates each complete response against the batches continue. PromptKit validates each complete response against the
validator-decision JSON Schema and performs configured structural repair before validator-decision JSON Schema and performs configured structural repair before
Audita receives decoded decisions. Audita then reconciles those decisions with Audita receives decoded decisions. Each actual batch presents candidates
the correction indices requested in that batch: one usable entry keeps its numbered from 1, independent of their internal proposal indices. Audita then
reconciles those local references: one usable entry keeps its
approval or editorial rejection; a missing or duplicate requested entry makes approval or editorial rejection; a missing or duplicate requested entry makes
only that candidate unavailable; unexpected indices are ignored. An empty list only that candidate unavailable; unexpected indices are ignored. An empty list
makes all requested candidates unavailable. Results are ordered by request makes all requested candidates unavailable. Decisions and unavailable indices
identity and satisfy strict internal cardinality before the runner advances map back to internal proposal identities; results are ordered by request identity
and satisfy strict internal cardinality before the runner advances
approved candidates. An unavailable candidate never advances as approved. approved candidates. An unavailable candidate never advances as approved.
Invalid confidence in a decoded test double also makes only its candidate Invalid confidence in a decoded test double also makes only its candidate
unavailable; production schema validation still rejects out-of-range confidence unavailable; production schema validation still rejects out-of-range confidence
@@ -79,6 +81,12 @@ chain without returning unchecked candidates as approved. Reconciliation makes
no semantic retry or extra model call; guided recovery remains no semantic retry or extra model call; guided recovery remains
[deferred](../roadmap/future.md#domain-aware-response-recovery). [deferred](../roadmap/future.md#domain-aware-response-recovery).
Debug request metadata records `proposal_indices` in local-reference order
(reference 1 is the first entry). Raw decoded responses and ignored indices use
local references; reconciled decisions and unavailable indices use internal
proposal indices. Batch token estimates use the same locally numbered payload
as prompt encoding.
## Decision and Rejection Reporting ## Decision and Rejection Reporting
Runner collects decisions, rejected candidates, and operation outcomes. Ledger Runner collects decisions, rejected candidates, and operation outcomes. Ledger
@@ -111,9 +119,11 @@ is covered by [report assembly](diagnostics-reporting.md).
## Supplemental Validation Context ## Supplemental Validation Context
Each candidate includes the immediately preceding and following source segments Each candidate includes the immediately preceding and following source segments
available within its fixed chunk, plus glossary entries whose names, aliases, or available within its fixed chunk, labeled `preceding` or `following` with speaker,
text, and categories, plus glossary entries whose names, aliases, or
recognized plurals appear in its original/resulting text or those neighbors. recognized plurals appear in its original/resulting text or those neighbors.
Glossary matching reuses the protected vocabulary; entries retain glossary order. Glossary matching reuses the protected vocabulary; entries retain glossary order.
Model-facing candidate and neighboring context omit internal segment IDs.
This supplies evidence such as a troupe definition for a proposed `Jesters` This supplies evidence such as a troupe definition for a proposed `Jesters`
correction or an adjacent speaker turn explaining deliberate repetition. A correction correction or an adjacent speaker turn explaining deliberate repetition. A correction
already clear from its own segment does not require such evidence. No audio, already clear from its own segment does not require such evidence. No audio,

View File

@@ -21,7 +21,8 @@ type transcriptSectionPayload struct {
} }
// MarshalTranscriptSectionJSON builds the standardized transcript-section JSON // MarshalTranscriptSectionJSON builds the standardized transcript-section JSON
// payload consumed by proposal prompt templates. // payload consumed by proposal prompt templates. IDs are one-based positions in
// this request; the transcript and its actual IDs are unchanged.
func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex int) ([]byte, error) { func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex int) ([]byte, error) {
payload := transcriptSectionPayload{ payload := transcriptSectionPayload{
SectionIndex: sectionIndex, SectionIndex: sectionIndex,
@@ -29,9 +30,9 @@ func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex in
} }
if transcript != nil { if transcript != nil {
for _, s := range transcript.Segments { for i, s := range transcript.Segments {
payload.Segments = append(payload.Segments, transcriptSectionSegment{ payload.Segments = append(payload.Segments, transcriptSectionSegment{
ID: s.ID, ID: i + 1,
Speaker: s.Speaker, Speaker: s.Speaker,
Start: s.Start, Start: s.Start,
End: s.End, End: s.End,

View File

@@ -10,7 +10,7 @@ import (
func TestMarshalTranscriptSectionJSONShape(t *testing.T) { func TestMarshalTranscriptSectionJSONShape(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{ transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}}, {ID: 817, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}},
}} }}
raw, err := MarshalTranscriptSectionJSON(transcript, 3) raw, err := MarshalTranscriptSectionJSON(transcript, 3)

View File

@@ -61,7 +61,7 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
generated, err := GenerateCandidates(context.Background(), Request{ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client}) generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@@ -36,7 +36,7 @@ type InteractionArtifacts struct {
// StructuredCorrectionProposal is one LLM response correction payload. // StructuredCorrectionProposal is one LLM response correction payload.
type StructuredCorrectionProposal struct { type StructuredCorrectionProposal struct {
TargetSegmentID int `json:"id"` SegmentReference int `json:"id"`
OriginalText string `json:"original_text"` OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"` CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"` Confidence float64 `json:"confidence"`
@@ -142,6 +142,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
"replacement_policy": req.ReplacementPolicy, "replacement_policy": req.ReplacementPolicy,
"section": req.Section, "section": req.Section,
"start_index": req.StartIndex, "start_index": req.StartIndex,
"segment_ids": segmentIDs(req.WorkingTranscript),
"prompt": completion.Prompt, "prompt": completion.Prompt,
} }
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap() requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
@@ -185,8 +186,13 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections)) corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections)) enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
for i, raw := range response.Corrections { for i, raw := range response.Corrections {
// Unknown references remain invalid candidates for the existing domain validators.
targetID := 0
if req.WorkingTranscript != nil && raw.SegmentReference > 0 && raw.SegmentReference <= len(req.WorkingTranscript.Segments) {
targetID = req.WorkingTranscript.Segments[raw.SegmentReference-1].ID
}
candidate := proposals.CorrectionProposal{ candidate := proposals.CorrectionProposal{
TargetSegmentID: raw.TargetSegmentID, TargetSegmentID: targetID,
OriginalText: raw.OriginalText, OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText, CorrectedText: raw.CorrectedText,
Confidence: raw.Confidence, Confidence: raw.Confidence,
@@ -247,3 +253,15 @@ func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata
ErrorPayloadPath: artifacts.ErrorPayloadPath, ErrorPayloadPath: artifacts.ErrorPayloadPath,
}, nil }, nil
} }
// segmentIDs records the request-local reference mapping in debug metadata.
func segmentIDs(transcript *schema.Transcript) []int {
if transcript == nil {
return nil
}
ids := make([]int, len(transcript.Segments))
for i, segment := range transcript.Segments {
ids[i] = segment.ID
}
return ids
}

View File

@@ -12,6 +12,7 @@ import (
ck "gitea.maximumdirect.net/eric/ConfigKit" ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
@@ -77,6 +78,7 @@ func defaultRequest(t *testing.T) Request {
PromptID: "modules.grammar.proposal", PromptID: "modules.grammar.proposal",
PromptVersion: "v1", PromptVersion: "v1",
Variables: map[string]string{"SectionJSON": "[]"}, Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
StartIndex: 0, StartIndex: 0,
} }
} }
@@ -86,7 +88,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
responses: []StructuredCorrectionSet{ responses: []StructuredCorrectionSet{
{ {
Corrections: []StructuredCorrectionProposal{ Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 7, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, {SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
}, },
}, },
}, },
@@ -104,6 +106,9 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
if len(got.Corrections) != 1 || len(got.Enriched) != 1 { if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
t.Fatalf("unexpected proposal lengths: %+v", got) t.Fatalf("unexpected proposal lengths: %+v", got)
} }
if got.Corrections[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Corrections)
}
if got.Enriched[0].ProposalIndex != 10 { if got.Enriched[0].ProposalIndex != 10 {
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex) t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
} }
@@ -118,7 +123,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) { func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
client := &fakeStructuredClient{ client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{ responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
}, },
} }
req := defaultRequest(t) req := defaultRequest(t)
@@ -140,7 +145,7 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) { func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredClient{ client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{ responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
}, },
} }
diag := &captureDiagnosticsWriter{} diag := &captureDiagnosticsWriter{}
@@ -172,7 +177,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
responses: []StructuredCorrectionSet{ responses: []StructuredCorrectionSet{
{ {
Corrections: []StructuredCorrectionProposal{ Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2}, {SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
}, },
}, },
}, },
@@ -235,8 +240,8 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) { func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{ baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{ Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9}, {SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
{TargetSegmentID: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9}, {SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
}, },
} }
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}} clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
@@ -266,8 +271,8 @@ func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
client := &fakeStructuredClient{ client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{ responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}}, {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 3, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}}, {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
}, },
} }
section0 := contracts.SectionMetadata{Index: 0} section0 := contracts.SectionMetadata{Index: 0}
@@ -276,6 +281,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
req0 := defaultRequest(t) req0 := defaultRequest(t)
req0.LLMClient = client req0.LLMClient = client
req0.Section = &section0 req0.Section = &section0
req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}}
req0.StartIndex = 0 req0.StartIndex = 0
part0, err := GenerateCandidates(context.Background(), req0) part0, err := GenerateCandidates(context.Background(), req0)
if err != nil { if err != nil {
@@ -285,6 +291,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
req1 := defaultRequest(t) req1 := defaultRequest(t)
req1.LLMClient = client req1.LLMClient = client
req1.Section = &section1 req1.Section = &section1
req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}}
req1.StartIndex = len(part0.Enriched) req1.StartIndex = len(part0.Enriched)
part1, err := GenerateCandidates(context.Background(), req1) part1, err := GenerateCandidates(context.Background(), req1)
if err != nil { if err != nil {
@@ -295,6 +302,9 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
if len(all) != 2 { if len(all) != 2 {
t.Fatalf("expected 2 proposals, got %d", len(all)) t.Fatalf("expected 2 proposals, got %d", len(all))
} }
if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 {
t.Fatalf("section references did not reset: %+v", all)
}
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 { if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex) t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
} }
@@ -308,7 +318,7 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "proposal-validation-secret" validationSecret := "proposal-validation-secret"
client := &fakeStructuredClient{ client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{ responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}}, {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
}, },
} }
t.Setenv("PRIMARY_SECRET", primarySecret) t.Setenv("PRIMARY_SECRET", primarySecret)

View File

@@ -2,6 +2,7 @@ package proposal_generation
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"testing" "testing"
@@ -17,7 +18,7 @@ import (
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) { func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
client := &fakeStructuredClient{ client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{ responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
}, },
} }
section := contracts.SectionMetadata{Index: 7} section := contracts.SectionMetadata{Index: 7}
@@ -135,3 +136,47 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
t.Fatalf("fatal operation lost: %+v, %v", result, err) t.Fatalf("fatal operation lost: %+v, %v", result, err)
} }
} }
func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}}
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9},
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
}}}}
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{WorkingTranscript: transcript},
RunSpec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique},
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal, BuildVariables: BuildProposalVariables,
})
if err != nil {
t.Fatal(err)
}
var payload struct {
Segments []struct {
ID int `json:"id"`
Text string `json:"text"`
} `json:"segments"`
}
if err := json.Unmarshal([]byte(client.calls[0].Variables["SectionJSON"]), &payload); err != nil {
t.Fatal(err)
}
if len(payload.Segments) != 2 || payload.Segments[0].ID != 1 || payload.Segments[1].ID != 2 || payload.Segments[1].Text != "uh hello" {
t.Fatalf("request references: %+v", payload)
}
if len(result.Proposals) != 4 {
t.Fatalf("proposals: %+v", result)
}
for i, id := range []int{903, 71, 903, 0} {
if result.Proposals[i].TargetSegmentID != id {
t.Fatalf("mapped proposal %d: %+v", i, result.Proposals[i])
}
}
if result.Proposals[0].CorrectedText != "" || transcript.Segments[0].ID != 71 || transcript.Segments[1].ID != 903 {
t.Fatal("mapping changed deletion or source identities")
}
}

View File

@@ -621,7 +621,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
if target, ok := out.(*validators.LLMValidationResponse); ok { if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{ *target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{ Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}, },
} }
} }
@@ -653,7 +653,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
target := out.(*validators.LLMValidationResponse) target := out.(*validators.LLMValidationResponse)
*target = validators.LLMValidationResponse{ *target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{ Validations: []validators.LLMValidationDecision{
{CorrectionIndex: section, Approved: true, Confidence: 0.99, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"},
}, },
} }
return contracts.StructuredCompletionResponse{}, nil return contracts.StructuredCompletionResponse{}, nil
@@ -942,7 +942,7 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
} }
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) { func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}}} client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}}}
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
if err != nil { if err != nil {
t.Fatalf("NewLLMBackedValidator: %v", err) t.Fatalf("NewLLMBackedValidator: %v", err)
@@ -969,7 +969,7 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
} }
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.95, Reason: "reject"}}}}} client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "reject"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
@@ -1028,8 +1028,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) { func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}, {CorrectionIndex: 3, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"}, {CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"},
}}}} }}}}
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
@@ -1095,8 +1095,8 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) { func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, {CorrectionIndex: 1, Approved: false, Confidence: 0.9, Reason: "dup"},
}}}} }}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1122,7 +1122,7 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
func TestRunnerLLMValidatorBatching(t *testing.T) { func TestRunnerLLMValidatorBatching(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{ client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}, {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}, {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
}} }}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
@@ -1159,7 +1159,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
primarySecret := "runner-primary-secret" primarySecret := "runner-primary-secret"
validationSecret := "runner-validation-secret" validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}} client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
t.Setenv("RUNNER_PRIMARY_KEY", primarySecret) t.Setenv("RUNNER_PRIMARY_KEY", primarySecret)
t.Setenv("RUNNER_VALIDATION_KEY", validationSecret) t.Setenv("RUNNER_VALIDATION_KEY", validationSecret)
@@ -1265,7 +1265,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{ responses: []proposal_generation.StructuredCorrectionSet{
{ {
Corrections: []proposal_generation.StructuredCorrectionProposal{ Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, {SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
}, },
}, },
}, },
@@ -1308,7 +1308,7 @@ func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{ responses: []proposal_generation.StructuredCorrectionSet{
{ {
Corrections: []proposal_generation.StructuredCorrectionProposal{ Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5}, {SegmentReference: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5},
}, },
}, },
}, },
@@ -1342,7 +1342,7 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{ responses: []proposal_generation.StructuredCorrectionSet{
{ {
Corrections: []proposal_generation.StructuredCorrectionProposal{ Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5}, {SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5},
}, },
}, },
}, },

View File

@@ -70,7 +70,7 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est
} }
func estimateBatchTokens(estimator chunking.TokenEstimator, batch []LLMValidationItem) (int, error) { func estimateBatchTokens(estimator chunking.TokenEstimator, batch []LLMValidationItem) (int, error) {
payload, err := json.Marshal(batch) payload, err := json.Marshal(localValidationItems(batch))
if err != nil { if err != nil {
return 0, fmt.Errorf("marshal batch payload: %w", err) return 0, fmt.Errorf("marshal batch payload: %w", err)
} }

View File

@@ -8,8 +8,8 @@ import (
// validationContext supplies adjacent source segments within the current chunk // validationContext supplies adjacent source segments within the current chunk
// and glossary definitions mentioned in the candidate or that local context. // and glossary definitions mentioned in the candidate or that local context.
// These fields travel in each item so batching accounts for their size. // These fields travel in each item so batching accounts for their size.
func validationContext(req Request, segmentID int, corrected string) ([]schema.Segment, []schema.GlossaryEntry) { func validationContext(req Request, segmentID int, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
var neighbors []schema.Segment var neighbors []NeighboringSegment
var source string var source string
if req.WorkingTranscript != nil { if req.WorkingTranscript != nil {
segments := req.WorkingTranscript.Segments segments := req.WorkingTranscript.Segments
@@ -19,10 +19,10 @@ func validationContext(req Request, segmentID int, corrected string) ([]schema.S
} }
source = segment.Text source = segment.Text
if i > 0 { if i > 0 {
neighbors = append(neighbors, segments[i-1]) neighbors = append(neighbors, neighboringSegment("preceding", segments[i-1]))
} }
if i+1 < len(segments) { if i+1 < len(segments) {
neighbors = append(neighbors, segments[i+1]) neighbors = append(neighbors, neighboringSegment("following", segments[i+1]))
} }
break break
} }
@@ -66,3 +66,7 @@ func fitValidationContext(item LLMValidationItem, budget int, estimator chunking
} }
} }
} }
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {
return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories}
}

View File

@@ -29,7 +29,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
t.Fatalf("payload: %+v immediate: %+v", payload, immediate) t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
} }
item := payload.Items[0] item := payload.Items[0]
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].ID != 1 || item.NeighboringSegments[1].ID != 3 || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" { if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
t.Fatalf("context: %+v", item) t.Fatalf("context: %+v", item)
} }
variables, err := BuildValidationVariables(payload.Items, "") variables, err := BuildValidationVariables(payload.Items, "")
@@ -40,6 +40,9 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil { if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if strings.Contains(variables["PayloadJSON"], `"id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
}
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Summary != "A traveling troupe" { if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Summary != "A traveling troupe" {
t.Fatalf("rendered context: %+v", rendered) t.Fatalf("rendered context: %+v", rendered)
} }
@@ -60,7 +63,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
enriched := base enriched := base
enriched.NeighboringSegments = []schema.Segment{{ID: 2, Text: strings.Repeat("long context ", 300)}} enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []schema.GlossaryEntry{{Name: "cat", Summary: strings.Repeat("long definition ", 300)}} enriched.GlossaryEntries = []schema.GlossaryEntry{{Name: "cat", Summary: strings.Repeat("long definition ", 300)}}
fitted := fitValidationContext(enriched, required, estimator) fitted := fitValidationContext(enriched, required, estimator)
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted}) tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})

View File

@@ -18,7 +18,7 @@ import (
) )
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":0,"approved":true,"confidence":0.9,"reason":"ok"}]}` valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
for _, tc := range []struct { for _, tc := range []struct {
name string name string
responses []string responses []string

View File

@@ -17,16 +17,17 @@ type StructuredCompletionRequest = structuredoutput.Request
type StructuredCompletionResponse = structuredoutput.Response type StructuredCompletionResponse = structuredoutput.Response
type StructuredLLMClient = structuredoutput.Client type StructuredLLMClient = structuredoutput.Client
// LLMValidationItem is one candidate correction payload passed to LLM validators. // LLMValidationItem retains the internal proposal index until prompt encoding.
// BuildValidationVariables substitutes a one-based reference for each batch.
type LLMValidationItem struct { type LLMValidationItem struct {
CorrectionIndex int `json:"correction_index"` CorrectionIndex int `json:"correction_index"`
SegmentID int `json:"id"` SegmentID int `json:"-"`
OriginalText string `json:"original_text"` OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"` CorrectedText string `json:"corrected_text"`
OriginalSegmentText string `json:"original_segment_text"` OriginalSegmentText string `json:"original_segment_text"`
CorrectedSegmentText string `json:"corrected_segment_text"` CorrectedSegmentText string `json:"corrected_segment_text"`
Categories []string `json:"categories,omitempty"` Categories []string `json:"categories,omitempty"`
NeighboringSegments []schema.Segment `json:"neighboring_segments,omitempty"` NeighboringSegments []NeighboringSegment `json:"neighboring_segments,omitempty"`
GlossaryEntries []schema.GlossaryEntry `json:"glossary_entries,omitempty"` GlossaryEntries []schema.GlossaryEntry `json:"glossary_entries,omitempty"`
} }
@@ -41,6 +42,7 @@ type LLMValidationRequest struct {
Items []LLMValidationItem `json:"corrections"` Items []LLMValidationItem `json:"corrections"`
} }
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
type LLMValidationDecision struct { type LLMValidationDecision struct {
CorrectionIndex int `json:"correction_index"` CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"` Approved bool `json:"approved"`
@@ -51,3 +53,11 @@ type LLMValidationDecision struct {
type LLMValidationResponse struct { type LLMValidationResponse struct {
Validations []LLMValidationDecision `json:"validations"` Validations []LLMValidationDecision `json:"validations"`
} }
// NeighboringSegment carries semantic context without internal transcript identity.
type NeighboringSegment struct {
Position string `json:"position"`
Speaker string `json:"speaker"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}

View File

@@ -2,6 +2,7 @@ package validators
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"strings" "strings"
"testing" "testing"
@@ -47,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
} }
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"} failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{ client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed}, {completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}} }}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{}) validator.SetTokenEstimator(perItemEstimator{})
@@ -72,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]} survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{ nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}} }}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal) nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{}) nextValidator.SetTokenEstimator(perItemEstimator{})
@@ -84,6 +85,18 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 { if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 {
t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err) t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err)
} }
for _, call := range append(client.calls, nextClient.calls...) {
var items []LLMValidationItem
if err := json.Unmarshal([]byte(call.Variables["PayloadJSON"]), &items); err != nil {
t.Fatal(err)
}
if len(items) != 1 || items[0].CorrectionIndex != 1 {
t.Fatalf("batch references must restart: %+v", items)
}
}
if !next.Decisions[0].Approved || !next.Decisions[1].Approved || next.Decisions[0].ProposalIndex != 0 || next.Decisions[1].ProposalIndex != 2 {
t.Fatalf("rebatching lost identities: %+v", next.Decisions)
}
} }
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) { func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
@@ -103,7 +116,7 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) { func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{ client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{ response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"}, {CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
}}, }},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}}, completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
@@ -124,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) { func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema") cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{ client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}}, {err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
}} }}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{}) validator.SetTokenEstimator(perItemEstimator{})

View File

@@ -8,7 +8,7 @@ import (
) )
func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptDescription string) (map[string]string, error) { func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptDescription string) (map[string]string, error) {
payloadJSON, err := json.MarshalIndent(validationPayload, "", " ") payloadJSON, err := json.MarshalIndent(localValidationItems(validationPayload), "", " ")
if err != nil { if err != nil {
return nil, fmt.Errorf("marshal validation payload: %w", err) return nil, fmt.Errorf("marshal validation payload: %w", err)
} }
@@ -17,3 +17,12 @@ func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptD
"PayloadJSON": string(payloadJSON), "PayloadJSON": string(payloadJSON),
}, nil }, nil
} }
// localValidationItems gives each actual batch its own short references.
func localValidationItems(items []LLMValidationItem) []LLMValidationItem {
local := append([]LLMValidationItem(nil), items...)
for i := range local {
local[i].CorrectionIndex = i + 1
}
return local
}

View File

@@ -163,6 +163,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
"validator_name": v.name, "validator_name": v.name,
"validator_type": v.validatorType, "validator_type": v.validatorType,
"batch_index": batch.BatchIndex, "batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items),
"prompt": completion.Prompt, "prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(), "response_schema": responseSchema.DiagnosticsMap(),
}, },
@@ -303,22 +304,18 @@ type validationReconciliation struct {
} }
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation { func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
expected := make(map[int]struct{}, len(items))
for _, item := range items {
expected[item.CorrectionIndex] = struct{}{}
}
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))} result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
seen := make(map[int][]LLMValidationDecision, len(items)) seen := make(map[int][]LLMValidationDecision, len(items))
for _, d := range response.Validations { for _, d := range response.Validations {
if _, ok := expected[d.CorrectionIndex]; !ok { if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) {
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex) result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
continue continue
} }
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d) seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
} }
for _, item := range items { for i, item := range items {
entries := seen[item.CorrectionIndex] entries := seen[i+1]
if len(entries) == 0 { if len(entries) == 0 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing")) result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex) result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)

View File

@@ -152,8 +152,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) { func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"}, {CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
}}}} }}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil { if err != nil {
@@ -224,13 +224,14 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
unavailable []int unavailable []int
ignored []int ignored []int
}{ }{
{"missing and extra", []LLMValidationDecision{approve(7), reject(18), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}}, {"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
{"conflicting duplicate", []LLMValidationDecision{approve(7), approve(11), reject(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, {"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"identical duplicate", []LLMValidationDecision{approve(7), approve(11), approve(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, {"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(18), approve(7), approve(99), approve(11)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}}, {"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil}, {"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(18), approve(11), approve(7)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil}, {"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(7), {CorrectionIndex: 11, Approved: true, Confidence: 1.2}, reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, {"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}} client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
@@ -337,7 +338,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) { func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}} }}}}
writer := &captureValidationDiagnosticsWriter{} writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
@@ -376,7 +377,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
} }
for _, tc := range cases { for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) { t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}} client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind) validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -401,7 +402,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) { func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable") cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 1, Reason: "ok"}}}}} req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause} req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error var reported error
req.ReportDiagnosticError = func(err error) { reported = err } req.ReportDiagnosticError = func(err error) { reported = err }

View File

@@ -93,7 +93,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
responses: []proposal_generation.StructuredCorrectionSet{ responses: []proposal_generation.StructuredCorrectionSet{
{ {
Corrections: []proposal_generation.StructuredCorrectionProposal{ Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94}, {SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94},
}, },
}, },
}, },

View File

@@ -42,7 +42,7 @@ func (f *fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.St
items, _ := payload["corrections"].([]map[string]any) items, _ := payload["corrections"].([]map[string]any)
for _, item := range items { for _, item := range items {
target.Corrections = append(target.Corrections, proposal_generation.StructuredCorrectionProposal{ target.Corrections = append(target.Corrections, proposal_generation.StructuredCorrectionProposal{
TargetSegmentID: item["id"].(int), SegmentReference: item["id"].(int),
OriginalText: item["original_text"].(string), OriginalText: item["original_text"].(string),
CorrectedText: item["corrected_text"].(string), CorrectedText: item["corrected_text"].(string),
Confidence: item["confidence"].(float64), Confidence: item["confidence"].(float64),

View File

@@ -94,7 +94,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
responses: []proposal_generation.StructuredCorrectionSet{ responses: []proposal_generation.StructuredCorrectionSet{
{ {
Corrections: []proposal_generation.StructuredCorrectionProposal{ Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94}, {SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94},
}, },
}, },
}, },

View File

@@ -94,7 +94,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
responses: []proposal_generation.StructuredCorrectionSet{ responses: []proposal_generation.StructuredCorrectionSet{
{ {
Corrections: []proposal_generation.StructuredCorrectionProposal{ Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "I I think", CorrectedText: "I think", Confidence: 0.94}, {SegmentReference: 1, OriginalText: "I I think", CorrectedText: "I think", Confidence: 0.94},
}, },
}, },
}, },

View File

@@ -19,16 +19,14 @@ messages:
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context. - If a segment includes categories, treat them as additional transcript context.
- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary. - Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.
- Use the exact id from the input segment. - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement. - Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Each returned correction must contain only id, original_text, corrected_text, and confidence.
- Do not return corrections where original_text and corrected_text are identical. - Do not return corrections where original_text and corrected_text are identical.
- Do not return speaker, start, or end fields.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}. - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .HardeningPolicy }} {{ .HardeningPolicy }}

View File

@@ -20,18 +20,16 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling. - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context. - If a segment includes categories, treat them as additional transcript context.
- Use the exact id from the input segment. - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text. - Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed. - A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed.
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
- Each returned correction must contain only id, original_text, corrected_text, and confidence.
- Do not return corrections where original_text and corrected_text are identical. - Do not return corrections where original_text and corrected_text are identical.
- Do not return speaker, start, or end fields.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}. - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .HardeningPolicy }} {{ .HardeningPolicy }}

View File

@@ -19,16 +19,14 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context. - If a segment includes categories, treat them as additional transcript context.
- Use the exact id from the input segment. - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text. - Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Each returned correction must contain only id, original_text, corrected_text, and confidence.
- Do not return corrections where original_text and corrected_text are identical. - Do not return corrections where original_text and corrected_text are identical.
- Do not return speaker, start, or end fields.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}. - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .HardeningPolicy }} {{ .HardeningPolicy }}

View File

@@ -22,18 +22,16 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context. - If a segment includes categories, treat them as additional transcript context.
- Use the exact id from the input segment. - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text. - Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result. - An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result.
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
- Each returned correction must contain only id, original_text, corrected_text, and confidence.
- Do not return corrections where original_text and corrected_text are identical. - Do not return corrections where original_text and corrected_text are identical.
- Do not return speaker, start, or end fields.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}. - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .HardeningPolicy }} {{ .HardeningPolicy }}

View File

@@ -9,7 +9,7 @@ messages:
Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision. Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.
Rules: Rules:
- Return exactly one validation decision for each supplied correction_index, including rejected corrections. Copy correction_index exactly; do not substitute the segment id, renumber entries, or invent indices. Return a JSON object with a validations array matching the supplied schema. - Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
- Approve editorial revisions that preserve the underlying meaning of the segment. - Approve editorial revisions that preserve the underlying meaning of the segment.
- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies. - Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.
- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning. - Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.
@@ -21,8 +21,7 @@ messages:
- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning. - Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
- An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty. - An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty.
- If supplied, neighboring_segments are adjacent source segments for context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- Each returned validation must contain only correction_index, approved, confidence, and reason.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. - confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .HardeningPolicy }} {{ .HardeningPolicy }}

View File

@@ -9,7 +9,7 @@ messages:
Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning. Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.
Rules: Rules:
- Return exactly one validation decision for each supplied correction_index, including rejected corrections. Copy correction_index exactly; do not substitute the segment id, renumber entries, or invent indices. Return a JSON object with a validations array matching the supplied schema. - Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment. - Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.
- Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning. - Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning.
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
@@ -17,8 +17,7 @@ messages:
- For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone. - For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone.
- Do not reject a correction merely because the literal written word changes. - Do not reject a correction merely because the literal written word changes.
- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals. - Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.
- If supplied, neighboring_segments are adjacent source segments for context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- Each returned validation must contain only correction_index, approved, confidence, and reason.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. - confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .HardeningPolicy }} {{ .HardeningPolicy }}

View File

@@ -9,15 +9,14 @@ messages:
Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction. Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.
Rules: Rules:
- Return exactly one validation decision for each supplied correction_index, including rejected corrections. Copy correction_index exactly; do not substitute the segment id, renumber entries, or invent indices. Return a JSON object with a validations array matching the supplied schema. - Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
- Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction. - Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.
- Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn". - Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn".
- Reject unrelated substitutions like changing "Lyra" to "Jesters". - Reject unrelated substitutions like changing "Lyra" to "Jesters".
- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical. - Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.
- For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription. - For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription.
- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions. - Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.
- If supplied, neighboring_segments are adjacent source segments for context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- Each returned validation must contain only correction_index, approved, confidence, and reason.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. - confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .HardeningPolicy }} {{ .HardeningPolicy }}