diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index c40aa3b..3d32018 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -143,12 +143,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { } if validation { if p.mode == "span-clean" || p.mode == "span-filter" { - content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}` + content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":2,"approved":true,"confidence":0.99,"reason":"ok"}]}` } else { - content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}` + content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}` } if p.mode == "span-negative" && validationCall == 2 { - content = `{"validations":[{"correction_index":0,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}` + content = `{"validations":[{"correction_index":1,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}` } } } @@ -159,10 +159,8 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { } } if validation && !strings.HasPrefix(p.mode, "span-") { - content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}` - if p.mode == "repair-filter-later-failure" { - content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}` - } else if p.mode == "invalid-decision-after-repair" { + content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}` + if p.mode == "invalid-decision-after-repair" { if validationCall == 1 { content = `{"wrong":true}` } else { @@ -173,12 +171,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { if validation && strings.HasPrefix(p.mode, "decision-") { decisions := make([]map[string]any, 0, len(requestedIndices)+2) for _, index := range requestedIndices { - if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2 { + if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 { continue } - approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 1) + approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2) decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"}) - if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 { + if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 4 { decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"}) } } @@ -606,9 +604,9 @@ func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse(t *testing.T) { full, survivor := 0, 0 for _, indices := range requests { switch { - case reflect.DeepEqual(indices, []int{0, 1, 2, 3}): + case reflect.DeepEqual(indices, []int{1, 2, 3, 4}): full++ - case reflect.DeepEqual(indices, []int{0}): + case reflect.DeepEqual(indices, []int{1}): survivor++ default: t.Fatalf("unexpected validation identities: %v", requests) @@ -662,7 +660,7 @@ func TestSubprocessReportsFilteredExtraValidatorDecision(t *testing.T) { } requests := f.assertDecisionRequestsUseOneAttempt(t) for _, indices := range requests { - if !reflect.DeepEqual(indices, []int{0}) { + if !reflect.DeepEqual(indices, []int{1}) { t.Fatalf("unexpected identities: %v", requests) } } diff --git a/docs/internal/modules.md b/docs/internal/modules.md index 33c82d8..9f3d846 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -42,7 +42,13 @@ transcript section, and trimmed description once for all four modules. PromptKit renders their embedded definitions from those variables and validates the correction-set schema before Audita maps proposals. Domain validators still check confidence, applicability, protected glossary terms, and replacement -policy. Shared context encoding lives in `internal/framework/promptcontext`. +policy. Model-facing segment IDs are one-based positions within the proposal +request. Generation maps returned references back to the current chunk's actual +segment IDs, preserving response order and multiple proposals per segment. +Unknown references become invalid-target candidates for the existing domain +checks; they are never interpreted as actual transcript IDs. Debug request +metadata records `segment_ids` in local-reference order (reference 1 is the +first entry). Shared context encoding lives in `internal/framework/promptcontext`. An empty `corrected_text` deletes the identified span when the resulting segment remains nonblank. Whitespace-only original spans can express spacing corrections; matching stays literal under each module's replacement policy. No automatic diff --git a/docs/internal/prompts.md b/docs/internal/prompts.md index 9b37d86..6be6e05 100644 --- a/docs/internal/prompts.md +++ b/docs/internal/prompts.md @@ -42,9 +42,14 @@ pass these values through the role-bound adapter. See the Proposal instructions describe literal matching, each module's replacement policy, source-based proposal identity, and confidence in whether an edit is warranted. -Validator instructions distinguish correction indices from segment IDs and confidence -in the verdict from the verdict itself. The grammar article exception and glossary -mishearing exception apply consistently across their instructions. +Proposal `id` and validator `correction_index` values are short, one-based +references scoped to each request, not internal transcript/proposal identities. +Mapping belongs to [proposal generation](modules.md) and +[validator reconciliation](validators.md). Validator instructions distinguish +confidence in the verdict from the verdict itself. Compact output examples +illustrate the supplied schema without requiring fixed JSON formatting or key +order. The grammar article exception and glossary mishearing exception apply +consistently across their instructions. Only the three active validator definitions are retained. The unused internal `grammar_review` and `spoken_word_review` definitions and type selections were diff --git a/docs/internal/validators.md b/docs/internal/validators.md index d751e85..060ab81 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -63,12 +63,14 @@ Oversized single inputs are rejected with a consolidated model failures make only their batch candidates unavailable while other batches continue. PromptKit validates each complete response against the validator-decision JSON Schema and performs configured structural repair before -Audita receives decoded decisions. Audita then reconciles those decisions with -the correction indices requested in that batch: one usable entry keeps its +Audita receives decoded decisions. Each actual batch presents candidates +numbered from 1, independent of their internal proposal indices. Audita then +reconciles those local references: one usable entry keeps its approval or editorial rejection; a missing or duplicate requested entry makes only that candidate unavailable; unexpected indices are ignored. An empty list -makes all requested candidates unavailable. Results are ordered by request -identity and satisfy strict internal cardinality before the runner advances +makes all requested candidates unavailable. Decisions and unavailable indices +map back to internal proposal identities; results are ordered by request identity +and satisfy strict internal cardinality before the runner advances approved candidates. An unavailable candidate never advances as approved. Invalid confidence in a decoded test double also makes only its candidate unavailable; production schema validation still rejects out-of-range confidence @@ -79,6 +81,12 @@ chain without returning unchecked candidates as approved. Reconciliation makes no semantic retry or extra model call; guided recovery remains [deferred](../roadmap/future.md#domain-aware-response-recovery). +Debug request metadata records `proposal_indices` in local-reference order +(reference 1 is the first entry). Raw decoded responses and ignored indices use +local references; reconciled decisions and unavailable indices use internal +proposal indices. Batch token estimates use the same locally numbered payload +as prompt encoding. + ## Decision and Rejection Reporting Runner collects decisions, rejected candidates, and operation outcomes. Ledger @@ -111,9 +119,11 @@ is covered by [report assembly](diagnostics-reporting.md). ## Supplemental Validation Context Each candidate includes the immediately preceding and following source segments -available within its fixed chunk, plus glossary entries whose names, aliases, or +available within its fixed chunk, labeled `preceding` or `following` with speaker, +text, and categories, plus glossary entries whose names, aliases, or recognized plurals appear in its original/resulting text or those neighbors. Glossary matching reuses the protected vocabulary; entries retain glossary order. +Model-facing candidate and neighboring context omit internal segment IDs. This supplies evidence such as a troupe definition for a proposed `Jesters` correction or an adjacent speaker turn explaining deliberate repetition. A correction already clear from its own segment does not require such evidence. No audio, diff --git a/internal/framework/promptcontext/transcript_section.go b/internal/framework/promptcontext/transcript_section.go index 81d3186..2cab50a 100644 --- a/internal/framework/promptcontext/transcript_section.go +++ b/internal/framework/promptcontext/transcript_section.go @@ -21,7 +21,8 @@ type transcriptSectionPayload struct { } // MarshalTranscriptSectionJSON builds the standardized transcript-section JSON -// payload consumed by proposal prompt templates. +// payload consumed by proposal prompt templates. IDs are one-based positions in +// this request; the transcript and its actual IDs are unchanged. func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex int) ([]byte, error) { payload := transcriptSectionPayload{ SectionIndex: sectionIndex, @@ -29,9 +30,9 @@ func MarshalTranscriptSectionJSON(transcript *schema.Transcript, sectionIndex in } if transcript != nil { - for _, s := range transcript.Segments { + for i, s := range transcript.Segments { payload.Segments = append(payload.Segments, transcriptSectionSegment{ - ID: s.ID, + ID: i + 1, Speaker: s.Speaker, Start: s.Start, End: s.End, diff --git a/internal/framework/promptcontext/transcript_section_test.go b/internal/framework/promptcontext/transcript_section_test.go index 062e8db..b15af91 100644 --- a/internal/framework/promptcontext/transcript_section_test.go +++ b/internal/framework/promptcontext/transcript_section_test.go @@ -10,7 +10,7 @@ import ( func TestMarshalTranscriptSectionJSONShape(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{ - {ID: 1, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}}, + {ID: 817, Speaker: "A", Start: 0.1, End: 1.2, Text: "alpha", Categories: []string{"session", "intro"}}, }} raw, err := MarshalTranscriptSectionJSON(transcript, 3) diff --git a/internal/framework/proposal_generation/adapter_integration_test.go b/internal/framework/proposal_generation/adapter_integration_test.go index 844580c..8af81c5 100644 --- a/internal/framework/proposal_generation/adapter_integration_test.go +++ b/internal/framework/proposal_generation/adapter_integration_test.go @@ -61,7 +61,7 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) { if err != nil { t.Fatal(err) } - generated, err := GenerateCandidates(context.Background(), Request{ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client}) + generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client}) if err != nil { t.Fatal(err) } diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index 21b84a5..4983790 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -36,10 +36,10 @@ type InteractionArtifacts struct { // StructuredCorrectionProposal is one LLM response correction payload. type StructuredCorrectionProposal struct { - TargetSegmentID int `json:"id"` - OriginalText string `json:"original_text"` - CorrectedText string `json:"corrected_text"` - Confidence float64 `json:"confidence"` + SegmentReference int `json:"id"` + OriginalText string `json:"original_text"` + CorrectedText string `json:"corrected_text"` + Confidence float64 `json:"confidence"` } // StructuredCorrectionSet is the reusable structured LLM response model for @@ -142,6 +142,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { "replacement_policy": req.ReplacementPolicy, "section": req.Section, "start_index": req.StartIndex, + "segment_ids": segmentIDs(req.WorkingTranscript), "prompt": completion.Prompt, } requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap() @@ -185,8 +186,13 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections)) enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections)) for i, raw := range response.Corrections { + // Unknown references remain invalid candidates for the existing domain validators. + targetID := 0 + if req.WorkingTranscript != nil && raw.SegmentReference > 0 && raw.SegmentReference <= len(req.WorkingTranscript.Segments) { + targetID = req.WorkingTranscript.Segments[raw.SegmentReference-1].ID + } candidate := proposals.CorrectionProposal{ - TargetSegmentID: raw.TargetSegmentID, + TargetSegmentID: targetID, OriginalText: raw.OriginalText, CorrectedText: raw.CorrectedText, Confidence: raw.Confidence, @@ -247,3 +253,15 @@ func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata ErrorPayloadPath: artifacts.ErrorPayloadPath, }, nil } + +// segmentIDs records the request-local reference mapping in debug metadata. +func segmentIDs(transcript *schema.Transcript) []int { + if transcript == nil { + return nil + } + ids := make([]int, len(transcript.Segments)) + for i, segment := range transcript.Segments { + ids[i] = segment.ID + } + return ids +} diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index fc85e3a..cd463c4 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -12,6 +12,7 @@ import ( ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/internal/core/config" + "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" @@ -77,6 +78,7 @@ func defaultRequest(t *testing.T) Request { PromptID: "modules.grammar.proposal", PromptVersion: "v1", Variables: map[string]string{"SectionJSON": "[]"}, + WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}}, StartIndex: 0, } } @@ -86,7 +88,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) { responses: []StructuredCorrectionSet{ { Corrections: []StructuredCorrectionProposal{ - {TargetSegmentID: 7, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, + {SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, }, }, }, @@ -104,6 +106,9 @@ func TestGenerateCandidatesSuccess(t *testing.T) { if len(got.Corrections) != 1 || len(got.Enriched) != 1 { t.Fatalf("unexpected proposal lengths: %+v", got) } + if got.Corrections[0].TargetSegmentID != 7 { + t.Fatalf("local reference not resolved: %+v", got.Corrections) + } if got.Enriched[0].ProposalIndex != 10 { t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex) } @@ -118,7 +123,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) { func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ - {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, + {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, }, } req := defaultRequest(t) @@ -140,7 +145,7 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) { func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ - {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, + {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, }, } diag := &captureDiagnosticsWriter{} @@ -172,7 +177,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes responses: []StructuredCorrectionSet{ { Corrections: []StructuredCorrectionProposal{ - {TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2}, + {SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2}, }, }, }, @@ -235,8 +240,8 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) { func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) { baseResponse := StructuredCorrectionSet{ Corrections: []StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9}, - {TargetSegmentID: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9}, + {SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9}, + {SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9}, }, } clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}} @@ -266,8 +271,8 @@ func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) { func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ - {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}}, - {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 3, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}}, + {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}}, + {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}}, }, } section0 := contracts.SectionMetadata{Index: 0} @@ -276,6 +281,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { req0 := defaultRequest(t) req0.LLMClient = client req0.Section = §ion0 + req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}} req0.StartIndex = 0 part0, err := GenerateCandidates(context.Background(), req0) if err != nil { @@ -285,6 +291,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { req1 := defaultRequest(t) req1.LLMClient = client req1.Section = §ion1 + req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}} req1.StartIndex = len(part0.Enriched) part1, err := GenerateCandidates(context.Background(), req1) if err != nil { @@ -295,6 +302,9 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { if len(all) != 2 { t.Fatalf("expected 2 proposals, got %d", len(all)) } + if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 { + t.Fatalf("section references did not reset: %+v", all) + } if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 { t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex) } @@ -308,7 +318,7 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) { validationSecret := "proposal-validation-secret" client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ - {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}}, + {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}}, }, } t.Setenv("PRIMARY_SECRET", primarySecret) diff --git a/internal/framework/proposal_generation/module_proposal_test.go b/internal/framework/proposal_generation/module_proposal_test.go index 4c2c77a..6fb49a9 100644 --- a/internal/framework/proposal_generation/module_proposal_test.go +++ b/internal/framework/proposal_generation/module_proposal_test.go @@ -2,6 +2,7 @@ package proposal_generation import ( "context" + "encoding/json" "errors" "testing" @@ -17,7 +18,7 @@ import ( func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ - {Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, + {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}}, }, } section := contracts.SectionMetadata{Index: 7} @@ -135,3 +136,47 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) { t.Fatalf("fatal operation lost: %+v, %v", result, err) } } + +func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) { + transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}} + client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{ + {SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9}, + {SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9}, + {SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9}, + {SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9}, + }}}} + result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{ + ProposalRequest: contracts.ProposalRequest{ + ExecutionContext: contracts.ExecutionContext{WorkingTranscript: transcript}, + RunSpec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique}, + LLMClient: client, + }, + PromptID: prompts.PromptIDModuleGrammarProposal, BuildVariables: BuildProposalVariables, + }) + if err != nil { + t.Fatal(err) + } + var payload struct { + Segments []struct { + ID int `json:"id"` + Text string `json:"text"` + } `json:"segments"` + } + if err := json.Unmarshal([]byte(client.calls[0].Variables["SectionJSON"]), &payload); err != nil { + t.Fatal(err) + } + if len(payload.Segments) != 2 || payload.Segments[0].ID != 1 || payload.Segments[1].ID != 2 || payload.Segments[1].Text != "uh hello" { + t.Fatalf("request references: %+v", payload) + } + if len(result.Proposals) != 4 { + t.Fatalf("proposals: %+v", result) + } + for i, id := range []int{903, 71, 903, 0} { + if result.Proposals[i].TargetSegmentID != id { + t.Fatalf("mapped proposal %d: %+v", i, result.Proposals[i]) + } + } + if result.Proposals[0].CorrectedText != "" || transcript.Segments[0].ID != 71 || transcript.Segments[1].ID != 903 { + t.Fatal("mapping changed deletion or source identities") + } +} diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 74c90da..4125ec3 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -621,7 +621,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context if target, ok := out.(*validators.LLMValidationResponse); ok { *target = validators.LLMValidationResponse{ Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, + {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}, }, } } @@ -653,7 +653,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req target := out.(*validators.LLMValidationResponse) *target = validators.LLMValidationResponse{ Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: section, Approved: true, Confidence: 0.99, Reason: "ok"}, + {CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"}, }, } return contracts.StructuredCompletionResponse{}, nil @@ -942,7 +942,7 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr } func TestRunnerLLMValidatorApprovalApplied(t *testing.T) { - client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}}} + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}}} llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) if err != nil { t.Fatalf("NewLLMBackedValidator: %v", err) @@ -969,7 +969,7 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) { } func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { - client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.95, Reason: "reject"}}}}} + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "reject"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { @@ -1028,8 +1028,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress( func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}, - {CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}, + {CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}, + {CorrectionIndex: 3, Approved: true, Confidence: .9, Reason: "ok"}, {CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"}, }}}} first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) @@ -1095,8 +1095,8 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) { func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, - {CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"}, + {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}, + {CorrectionIndex: 1, Approved: false, Confidence: 0.9, Reason: "dup"}, }}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ @@ -1122,7 +1122,7 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) } func TestRunnerLLMValidatorBatching(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{ - {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}, {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}, }} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) @@ -1159,7 +1159,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) { func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { primarySecret := "runner-primary-secret" validationSecret := "runner-validation-secret" - client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}} + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) t.Setenv("RUNNER_PRIMARY_KEY", primarySecret) t.Setenv("RUNNER_VALIDATION_KEY", validationSecret) @@ -1265,7 +1265,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) { responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, + {SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, }, }, }, @@ -1308,7 +1308,7 @@ func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) { responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5}, + {SegmentReference: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5}, }, }, }, @@ -1342,7 +1342,7 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) { responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5}, + {SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5}, }, }, }, diff --git a/internal/framework/validators/llm_batching.go b/internal/framework/validators/llm_batching.go index f5cfa9b..3fc805f 100644 --- a/internal/framework/validators/llm_batching.go +++ b/internal/framework/validators/llm_batching.go @@ -70,7 +70,7 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est } func estimateBatchTokens(estimator chunking.TokenEstimator, batch []LLMValidationItem) (int, error) { - payload, err := json.Marshal(batch) + payload, err := json.Marshal(localValidationItems(batch)) if err != nil { return 0, fmt.Errorf("marshal batch payload: %w", err) } diff --git a/internal/framework/validators/llm_context.go b/internal/framework/validators/llm_context.go index cac7461..618134c 100644 --- a/internal/framework/validators/llm_context.go +++ b/internal/framework/validators/llm_context.go @@ -8,8 +8,8 @@ import ( // validationContext supplies adjacent source segments within the current chunk // and glossary definitions mentioned in the candidate or that local context. // These fields travel in each item so batching accounts for their size. -func validationContext(req Request, segmentID int, corrected string) ([]schema.Segment, []schema.GlossaryEntry) { - var neighbors []schema.Segment +func validationContext(req Request, segmentID int, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) { + var neighbors []NeighboringSegment var source string if req.WorkingTranscript != nil { segments := req.WorkingTranscript.Segments @@ -19,10 +19,10 @@ func validationContext(req Request, segmentID int, corrected string) ([]schema.S } source = segment.Text if i > 0 { - neighbors = append(neighbors, segments[i-1]) + neighbors = append(neighbors, neighboringSegment("preceding", segments[i-1])) } if i+1 < len(segments) { - neighbors = append(neighbors, segments[i+1]) + neighbors = append(neighbors, neighboringSegment("following", segments[i+1])) } break } @@ -66,3 +66,7 @@ func fitValidationContext(item LLMValidationItem, budget int, estimator chunking } } } + +func neighboringSegment(position string, segment schema.Segment) NeighboringSegment { + return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories} +} diff --git a/internal/framework/validators/llm_context_test.go b/internal/framework/validators/llm_context_test.go index 5a940c9..fe7ed25 100644 --- a/internal/framework/validators/llm_context_test.go +++ b/internal/framework/validators/llm_context_test.go @@ -29,7 +29,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi t.Fatalf("payload: %+v immediate: %+v", payload, immediate) } item := payload.Items[0] - if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].ID != 1 || item.NeighboringSegments[1].ID != 3 || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" { + if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" { t.Fatalf("context: %+v", item) } variables, err := BuildValidationVariables(payload.Items, "") @@ -40,6 +40,9 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil { t.Fatal(err) } + if strings.Contains(variables["PayloadJSON"], `"id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 { + t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"]) + } if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Summary != "A traveling troupe" { t.Fatalf("rendered context: %+v", rendered) } @@ -60,7 +63,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) { t.Fatal(err) } enriched := base - enriched.NeighboringSegments = []schema.Segment{{ID: 2, Text: strings.Repeat("long context ", 300)}} + enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}} enriched.GlossaryEntries = []schema.GlossaryEntry{{Name: "cat", Summary: strings.Repeat("long definition ", 300)}} fitted := fitValidationContext(enriched, required, estimator) tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted}) diff --git a/internal/framework/validators/llm_integration_test.go b/internal/framework/validators/llm_integration_test.go index 212bffa..a2eadef 100644 --- a/internal/framework/validators/llm_integration_test.go +++ b/internal/framework/validators/llm_integration_test.go @@ -18,7 +18,7 @@ import ( ) func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { - valid := `{"validations":[{"correction_index":0,"approved":true,"confidence":0.9,"reason":"ok"}]}` + valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}` for _, tc := range []struct { name string responses []string diff --git a/internal/framework/validators/llm_models.go b/internal/framework/validators/llm_models.go index 8fd1c75..350e331 100644 --- a/internal/framework/validators/llm_models.go +++ b/internal/framework/validators/llm_models.go @@ -17,16 +17,17 @@ type StructuredCompletionRequest = structuredoutput.Request type StructuredCompletionResponse = structuredoutput.Response type StructuredLLMClient = structuredoutput.Client -// LLMValidationItem is one candidate correction payload passed to LLM validators. +// LLMValidationItem retains the internal proposal index until prompt encoding. +// BuildValidationVariables substitutes a one-based reference for each batch. type LLMValidationItem struct { CorrectionIndex int `json:"correction_index"` - SegmentID int `json:"id"` + SegmentID int `json:"-"` OriginalText string `json:"original_text"` CorrectedText string `json:"corrected_text"` OriginalSegmentText string `json:"original_segment_text"` CorrectedSegmentText string `json:"corrected_segment_text"` Categories []string `json:"categories,omitempty"` - NeighboringSegments []schema.Segment `json:"neighboring_segments,omitempty"` + NeighboringSegments []NeighboringSegment `json:"neighboring_segments,omitempty"` GlossaryEntries []schema.GlossaryEntry `json:"glossary_entries,omitempty"` } @@ -41,6 +42,7 @@ type LLMValidationRequest struct { Items []LLMValidationItem `json:"corrections"` } +// LLMValidationDecision contains a request-local correction_index, not a proposal index. type LLMValidationDecision struct { CorrectionIndex int `json:"correction_index"` Approved bool `json:"approved"` @@ -51,3 +53,11 @@ type LLMValidationDecision struct { type LLMValidationResponse struct { Validations []LLMValidationDecision `json:"validations"` } + +// NeighboringSegment carries semantic context without internal transcript identity. +type NeighboringSegment struct { + Position string `json:"position"` + Speaker string `json:"speaker"` + Text string `json:"text"` + Categories []string `json:"categories,omitempty"` +} diff --git a/internal/framework/validators/llm_partial_test.go b/internal/framework/validators/llm_partial_test.go index a54e6ea..ac6c41d 100644 --- a/internal/framework/validators/llm_partial_test.go +++ b/internal/framework/validators/llm_partial_test.go @@ -2,6 +2,7 @@ package validators import ( "context" + "encoding/json" "errors" "strings" "testing" @@ -47,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { } failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"} client := &scriptedValidationClient{replies: []validationReply{ - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, {completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed}, - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, }} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) @@ -72,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]} nextClient := &scriptedValidationClient{replies: []validationReply{ - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, }} nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal) nextValidator.SetTokenEstimator(perItemEstimator{}) @@ -84,6 +85,18 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 { t.Fatalf("later validator rebatching: %+v, calls=%+v, err=%v", next, nextClient.calls, err) } + for _, call := range append(client.calls, nextClient.calls...) { + var items []LLMValidationItem + if err := json.Unmarshal([]byte(call.Variables["PayloadJSON"]), &items); err != nil { + t.Fatal(err) + } + if len(items) != 1 || items[0].CorrectionIndex != 1 { + t.Fatalf("batch references must restart: %+v", items) + } + } + if !next.Decisions[0].Approved || !next.Decisions[1].Approved || next.Decisions[0].ProposalIndex != 0 || next.Decisions[1].ProposalIndex != 2 { + t.Fatalf("rebatching lost identities: %+v", next.Decisions) + } } func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) { @@ -103,7 +116,7 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) { func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) { client := &scriptedValidationClient{replies: []validationReply{{ response: LLMValidationResponse{Validations: []LLMValidationDecision{ - {CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}, + {CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}, {CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"}, }}, completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}}, @@ -124,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) { func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) { cause := errors.New("broken embedded schema") client := &scriptedValidationClient{replies: []validationReply{ - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, {err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}}, - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 2, Approved: true, Confidence: .9, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}, }} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) diff --git a/internal/framework/validators/llm_prompt_variables.go b/internal/framework/validators/llm_prompt_variables.go index 60f96fb..c27edde 100644 --- a/internal/framework/validators/llm_prompt_variables.go +++ b/internal/framework/validators/llm_prompt_variables.go @@ -8,7 +8,7 @@ import ( ) func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptDescription string) (map[string]string, error) { - payloadJSON, err := json.MarshalIndent(validationPayload, "", " ") + payloadJSON, err := json.MarshalIndent(localValidationItems(validationPayload), "", " ") if err != nil { return nil, fmt.Errorf("marshal validation payload: %w", err) } @@ -17,3 +17,12 @@ func BuildValidationVariables(validationPayload []LLMValidationItem, transcriptD "PayloadJSON": string(payloadJSON), }, nil } + +// localValidationItems gives each actual batch its own short references. +func localValidationItems(items []LLMValidationItem) []LLMValidationItem { + local := append([]LLMValidationItem(nil), items...) + for i := range local { + local[i].CorrectionIndex = i + 1 + } + return local +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index d260d68..19078c0 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -160,11 +160,12 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction( stage, map[string]any{ - "validator_name": v.name, - "validator_type": v.validatorType, - "batch_index": batch.BatchIndex, - "prompt": completion.Prompt, - "response_schema": responseSchema.DiagnosticsMap(), + "validator_name": v.name, + "validator_type": v.validatorType, + "batch_index": batch.BatchIndex, + "proposal_indices": batchProposalIndices(batch.Items), + "prompt": completion.Prompt, + "response_schema": responseSchema.DiagnosticsMap(), }, completion.Prepared, responsePayload, @@ -303,22 +304,18 @@ type validationReconciliation struct { } func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation { - expected := make(map[int]struct{}, len(items)) - for _, item := range items { - expected[item.CorrectionIndex] = struct{}{} - } result := validationReconciliation{Decisions: make([]Decision, 0, len(items))} seen := make(map[int][]LLMValidationDecision, len(items)) for _, d := range response.Validations { - if _, ok := expected[d.CorrectionIndex]; !ok { + if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) { result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex) continue } seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d) } - for _, item := range items { - entries := seen[item.CorrectionIndex] + for i, item := range items { + entries := seen[i+1] if len(entries) == 0 { result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonMissingValidationDecision, "required model validation decision was missing")) result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex) diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index c66d7b8..9bf96dd 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -152,8 +152,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) { func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ - {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, - {CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"}, + {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}, + {CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"}, }}}} v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) if err != nil { @@ -224,13 +224,14 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) { unavailable []int ignored []int }{ - {"missing and extra", []LLMValidationDecision{approve(7), reject(18), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}}, - {"conflicting duplicate", []LLMValidationDecision{approve(7), approve(11), reject(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, - {"identical duplicate", []LLMValidationDecision{approve(7), approve(11), approve(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, - {"extra only", []LLMValidationDecision{approve(99), reject(18), approve(7), approve(99), approve(11)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}}, + {"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}}, + {"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}}, + {"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, + {"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, + {"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}}, {"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil}, - {"reordered", []LLMValidationDecision{reject(18), approve(11), approve(7)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil}, - {"invalid confidence", []LLMValidationDecision{approve(7), {CorrectionIndex: 11, Approved: true, Confidence: 1.2}, reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, + {"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil}, + {"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, } { t.Run(tc.name, func(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}} @@ -337,7 +338,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ - {CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"}, + {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}, }}}} writer := &captureValidationDiagnosticsWriter{} v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) @@ -376,7 +377,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) { } for _, tc := range cases { t.Run(string(tc.kind), func(t *testing.T) { - client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}} + client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}} validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind) if err != nil { t.Fatal(err) @@ -401,7 +402,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) { func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) { cause := errors.New("diagnostic storage unavailable") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) - req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 1, Reason: "ok"}}}}} + req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}} req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause} var reported error req.ReportDiagnosticError = func(err error) { reported = err } diff --git a/internal/modules/glossary/module_test.go b/internal/modules/glossary/module_test.go index 68a0d63..64d68e0 100644 --- a/internal/modules/glossary/module_test.go +++ b/internal/modules/glossary/module_test.go @@ -93,7 +93,7 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94}, + {SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94}, }, }, }, diff --git a/internal/modules/grammar/module_test.go b/internal/modules/grammar/module_test.go index 812c74f..e42e2d8 100644 --- a/internal/modules/grammar/module_test.go +++ b/internal/modules/grammar/module_test.go @@ -42,10 +42,10 @@ func (f *fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.St items, _ := payload["corrections"].([]map[string]any) for _, item := range items { target.Corrections = append(target.Corrections, proposal_generation.StructuredCorrectionProposal{ - TargetSegmentID: item["id"].(int), - OriginalText: item["original_text"].(string), - CorrectedText: item["corrected_text"].(string), - Confidence: item["confidence"].(float64), + SegmentReference: item["id"].(int), + OriginalText: item["original_text"].(string), + CorrectedText: item["corrected_text"].(string), + Confidence: item["confidence"].(float64), }) } return contracts.StructuredCompletionResponse{}, nil diff --git a/internal/modules/homophones/module_test.go b/internal/modules/homophones/module_test.go index 986ac63..305826f 100644 --- a/internal/modules/homophones/module_test.go +++ b/internal/modules/homophones/module_test.go @@ -94,7 +94,7 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94}, + {SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94}, }, }, }, diff --git a/internal/modules/spoken_word/module_test.go b/internal/modules/spoken_word/module_test.go index 937fe0f..e57570f 100644 --- a/internal/modules/spoken_word/module_test.go +++ b/internal/modules/spoken_word/module_test.go @@ -94,7 +94,7 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ - {TargetSegmentID: 1, OriginalText: "I I think", CorrectedText: "I think", Confidence: 0.94}, + {SegmentReference: 1, OriginalText: "I I think", CorrectedText: "I think", Confidence: 0.94}, }, }, }, diff --git a/internal/prompts/assets/prompts/modules.glossary.proposal.yaml b/internal/prompts/assets/prompts/modules.glossary.proposal.yaml index 9ae15f8..4b076de 100644 --- a/internal/prompts/assets/prompts/modules.glossary.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.glossary.proposal.yaml @@ -19,16 +19,14 @@ messages: - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - If a segment includes categories, treat them as additional transcript context. - Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary. - - Use the exact id from the input segment. + - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target. - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - - Each returned correction must contain only id, original_text, corrected_text, and confidence. - Do not return corrections where original_text and corrected_text are identical. - - Do not return speaker, start, or end fields. - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - - Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}. + - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. {{ .HardeningPolicy }} diff --git a/internal/prompts/assets/prompts/modules.grammar.proposal.yaml b/internal/prompts/assets/prompts/modules.grammar.proposal.yaml index aeaa6ef..c59a6ea 100644 --- a/internal/prompts/assets/prompts/modules.grammar.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.grammar.proposal.yaml @@ -20,18 +20,16 @@ messages: - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - If a segment includes categories, treat them as additional transcript context. - - Use the exact id from the input segment. + - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target. - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Choose an original_text span that appears exactly once in the current segment text. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed. - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - - Each returned correction must contain only id, original_text, corrected_text, and confidence. - Do not return corrections where original_text and corrected_text are identical. - - Do not return speaker, start, or end fields. - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - - Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}. + - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. {{ .HardeningPolicy }} diff --git a/internal/prompts/assets/prompts/modules.homophones.proposal.yaml b/internal/prompts/assets/prompts/modules.homophones.proposal.yaml index 3722d9f..d32cdc6 100644 --- a/internal/prompts/assets/prompts/modules.homophones.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.homophones.proposal.yaml @@ -19,16 +19,14 @@ messages: - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - If a segment includes categories, treat them as additional transcript context. - - Use the exact id from the input segment. + - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target. - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Choose an original_text span that appears exactly once in the current segment text. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - - Each returned correction must contain only id, original_text, corrected_text, and confidence. - Do not return corrections where original_text and corrected_text are identical. - - Do not return speaker, start, or end fields. - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - - Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}. + - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. {{ .HardeningPolicy }} diff --git a/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml b/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml index d788a32..d56ef47 100644 --- a/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml @@ -22,18 +22,16 @@ messages: - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - If a segment includes categories, treat them as additional transcript context. - - Use the exact id from the input segment. + - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target. - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Choose an original_text span that appears exactly once in the current segment text. - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result. - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - - Each returned correction must contain only id, original_text, corrected_text, and confidence. - Do not return corrections where original_text and corrected_text are identical. - - Do not return speaker, start, or end fields. - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - - Return a JSON object with a corrections array, matching the supplied schema. If no corrections are needed, return {"corrections": []}. + - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. {{ .HardeningPolicy }} diff --git a/internal/prompts/assets/prompts/validators.editorial_review.yaml b/internal/prompts/assets/prompts/validators.editorial_review.yaml index 1b9ae00..11de151 100644 --- a/internal/prompts/assets/prompts/validators.editorial_review.yaml +++ b/internal/prompts/assets/prompts/validators.editorial_review.yaml @@ -9,7 +9,7 @@ messages: Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision. Rules: - - Return exactly one validation decision for each supplied correction_index, including rejected corrections. Copy correction_index exactly; do not substitute the segment id, renumber entries, or invent indices. Return a JSON object with a validations array matching the supplied schema. + - Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}. - Approve editorial revisions that preserve the underlying meaning of the segment. - Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies. - Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning. @@ -21,8 +21,7 @@ messages: - Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning. - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. - An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty. - - If supplied, neighboring_segments are adjacent source segments for context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - - Each returned validation must contain only correction_index, approved, confidence, and reason. + - If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. {{ .HardeningPolicy }} diff --git a/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml b/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml index 899bc04..113321f 100644 --- a/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml +++ b/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml @@ -9,7 +9,7 @@ messages: Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning. Rules: - - Return exactly one validation decision for each supplied correction_index, including rejected corrections. Copy correction_index exactly; do not substitute the segment id, renumber entries, or invent indices. Return a JSON object with a validations array matching the supplied schema. + - Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}. - Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment. - Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning. - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. @@ -17,8 +17,7 @@ messages: - For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone. - Do not reject a correction merely because the literal written word changes. - Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals. - - If supplied, neighboring_segments are adjacent source segments for context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - - Each returned validation must contain only correction_index, approved, confidence, and reason. + - If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. {{ .HardeningPolicy }} diff --git a/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml b/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml index 81e8d5f..7b4fdfa 100644 --- a/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml +++ b/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml @@ -9,15 +9,14 @@ messages: Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction. Rules: - - Return exactly one validation decision for each supplied correction_index, including rejected corrections. Copy correction_index exactly; do not substitute the segment id, renumber entries, or invent indices. Return a JSON object with a validations array matching the supplied schema. + - Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}. - Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction. - Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn". - Reject unrelated substitutions like changing "Lyra" to "Jesters". - Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical. - For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription. - Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions. - - If supplied, neighboring_segments are adjacent source segments for context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - - Each returned validation must contain only correction_index, approved, confidence, and reason. + - If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. {{ .HardeningPolicy }}