From 07a4b655706da87619b276c601a87c144d287cee Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 25 Sep 2026 13:26:51 +0000 Subject: [PATCH] Share prompt instructions and simplify validation helpers --- docs/internal/prompts.md | 9 ++- docs/internal/validators.md | 10 +++- docs/roadmap/remediation.md | 29 ++++++++-- internal/framework/llm/adapter.go | 4 +- internal/framework/outcomes/outcomes.go | 13 +++++ .../framework/proposal_generation/generate.go | 7 +-- internal/framework/validators/llm_context.go | 30 ++++------ .../framework/validators/llm_context_test.go | 16 ++++++ .../framework/validators/llm_validators.go | 55 ++++++++++--------- .../prompts/modules.glossary.proposal.yaml | 8 +-- .../prompts/modules.grammar.proposal.yaml | 8 +-- .../prompts/modules.homophones.proposal.yaml | 8 +-- .../prompts/modules.spoken_word.proposal.yaml | 8 +-- .../prompts/validators.editorial_review.yaml | 6 +- .../validators.meaning_reversal_review.yaml | 6 +- .../validators.spoken_form_plausibility.yaml | 6 +- .../assets/shared/proposal_contract.md | 6 ++ .../assets/shared/validation_contract.md | 3 + internal/prompts/registry.go | 19 +++++-- internal/prompts/registry_test.go | 21 +++++++ 20 files changed, 172 insertions(+), 100 deletions(-) create mode 100644 internal/prompts/assets/shared/proposal_contract.md create mode 100644 internal/prompts/assets/shared/validation_contract.md diff --git a/docs/internal/prompts.md b/docs/internal/prompts.md index 6be6e05..c0015ef 100644 --- a/docs/internal/prompts.md +++ b/docs/internal/prompts.md @@ -13,9 +13,12 @@ PromptKit reads the embedded definitions and renders their system and user messages from named string variables. The role-bound adapter supplies -`HardeningPolicy` from the single embedded `assets/shared/prompt_hardening.md` -fragment after caller variables, so callers cannot replace that policy. Proposal prompts receive the -section JSON, glossary JSON where used, and transcript description block. +`HardeningPolicy` alongside `ProposalContract` and `ValidationContract` from +embedded fragments under `assets/shared`, after caller variables so callers cannot replace those +instructions. Common literal-span and identity guidance lives in the proposal +fragment; shared candidate-reference, context, and confidence guidance lives in +the validation fragment. Module-specific policies and examples remain in YAML. +Proposal prompts receive the section JSON, glossary JSON where used, and transcript description block. Validator prompts receive the correction payload and description block. The prompt assets retain the prior task instructions and output-schema association. The role-bound runtime supplies the complete JSON Schema output contract diff --git a/docs/internal/validators.md b/docs/internal/validators.md index 060ab81..0a2daba 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -77,8 +77,10 @@ unavailable; production schema validation still rejects out-of-range confidence before reconciliation. A partially unavailable batch records one failed operation with only those requested indices, while a successful batch with ignored extras records a decision-filtering recovery. Fatal errors stop the -chain without returning unchecked candidates as approved. Reconciliation makes -no semantic retry or extra model call; guided recovery remains +chain without returning unchecked candidates as approved. A local result builder +retains already-collected decisions and operations on early returns; full +cardinality checks and final sorting remain explicit on completed paths. +Reconciliation makes no semantic retry or extra model call; guided recovery remains [deferred](../roadmap/future.md#domain-aware-response-recovery). Debug request metadata records `proposal_indices` in local-reference order @@ -118,6 +120,10 @@ is covered by [report assembly](diagnostics-reporting.md). ## Supplemental Validation Context +Request construction prepares one segment-position lookup and protected vocabulary +for all candidates. Glossary selection remains candidate-specific, including the +proposed replacement text. + Each candidate includes the immediately preceding and following source segments available within its fixed chunk, labeled `preceding` or `following` with speaker, text, and categories, plus glossary entries whose names, aliases, or diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 900adc3..84fb5aa 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -3,7 +3,7 @@ **Reviewed:** 2026-09-25, source commit `011b021`. **Scope:** Recent partial-decision reconciliation, validation context, prompt alignment, and request-local reference changes, including related existing code. -**Status:** R27–R30 open. Previous findings and resolution records remain in Git +**Status:** R27–R30 addressed; see the resolution record below. Previous findings and resolution records remain in Git history through `011b021`. ## Architecture Assessment @@ -133,10 +133,29 @@ material gap exposed by the refactor. - This is a source-level maintainability audit, not a finding of demonstrated production failures or a measured performance assessment. -- No application tests were rerun for the read-only audit. This roadmap update - requires documentation link and whitespace checks only. +- No application tests were rerun for the original read-only audit; its roadmap + update used documentation link and whitespace checks. Implementation validation + is recorded below. - Future remediation should run focused behavioral tests and the broader suite appropriate to the actual change, following the testing policy. No live-provider evaluation or new evaluation framework is required by these recommendations. -- This document records recommendations only; no implementation, prompt, or - release changes are included in this update. +- The findings above describe the audited baseline. Implementation and validation + are recorded below; no release publication is included. + +## Resolution Record + +Implemented as focused changes within the existing packages. Proposal and validator +reference mapping and distinct protected-term policies remain separate. + +| Finding | Resolution and evidence | +| --- | --- | +| R27 | Common proposal and validation instructions live in two embedded shared fragments, supplied through the existing role-bound PromptKit adapter. Individual YAML definitions retain policy differences and examples. Rendering tests check fragment inclusion once per applicable prompt and representative module-specific guidance. | +| R28 | Validation request construction builds one segment-position lookup and protected vocabulary, reused by context selection for every candidate. A new test verifies that two replacements of the same source span receive their own relevant glossary entries. Existing neighbor, glossary-order, chunk-boundary, and budget tests pass. | +| R29 | Both proposal generation and validation use `outcomes.ExecutionRecoveries`. Call sites still control success and domain-specific recovery reasons. Existing tests cover combined recoveries and failed validation with repair counts but no recovery record. | +| R30 | A local result builder combines collected decisions and operation outcomes for partial returns and completed validation. Sorting and full cardinality checks remain outside it. Existing fatal-batch, partial-progress, runner, and reporting tests pass. | + +Validation: `go test ./...` passed, including PromptKit preparation, partial-decision +reconciliation, subprocess reports, and fatal-progress coverage. Documentation +links and whitespace checks passed. No live model evaluation or performance +measurement was performed; the changes make no new model-quality or latency +promise. diff --git a/internal/framework/llm/adapter.go b/internal/framework/llm/adapter.go index 31aa027..ee1c580 100644 --- a/internal/framework/llm/adapter.go +++ b/internal/framework/llm/adapter.go @@ -98,7 +98,9 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou for key, value := range req.Variables { vars[key] = value } - vars["HardeningPolicy"] = prompts.HardeningText() + for key, value := range prompts.SharedVariables() { + vars[key] = value + } apiKey := "" if backend.Credential != nil { apiKey = backend.Credential.Value() diff --git a/internal/framework/outcomes/outcomes.go b/internal/framework/outcomes/outcomes.go index f887696..74e6972 100644 --- a/internal/framework/outcomes/outcomes.go +++ b/internal/framework/outcomes/outcomes.go @@ -92,3 +92,16 @@ const ( WorkCompleted WorkState = "completed" WorkInterrupted WorkState = "interrupted" ) + +// ExecutionRecoveries classifies execution counts; callers decide whether their +// operation succeeded before recording these reasons. +func ExecutionRecoveries(counts ExecutionCounts) []RecoveryReason { + var reasons []RecoveryReason + if counts.StructuralRepairs > 0 { + reasons = append(reasons, RecoveryOutputRepaired) + } + if counts.TransientRetries > 0 { + reasons = append(reasons, RecoveryServiceRecovered) + } + return reasons +} diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index 4983790..020bdec 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -176,12 +176,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { return result, fmt.Errorf("proposal generation completion failed: %w", callErr) } operation.Succeeded = true - if completion.Execution.StructuralRepairs > 0 { - operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired) - } - if completion.Execution.TransientRetries > 0 { - operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered) - } + operation.RecoveryReasons = outcomes.ExecutionRecoveries(completion.Execution) corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections)) enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections)) diff --git a/internal/framework/validators/llm_context.go b/internal/framework/validators/llm_context.go index 618134c..69b0063 100644 --- a/internal/framework/validators/llm_context.go +++ b/internal/framework/validators/llm_context.go @@ -8,39 +8,29 @@ import ( // validationContext supplies adjacent source segments within the current chunk // and glossary definitions mentioned in the candidate or that local context. // These fields travel in each item so batching accounts for their size. -func validationContext(req Request, segmentID int, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) { +func validationContext(segments []schema.Segment, position int, glossary *schema.Glossary, vocabulary ProtectedVocabulary, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) { var neighbors []NeighboringSegment - var source string - if req.WorkingTranscript != nil { - segments := req.WorkingTranscript.Segments - for i, segment := range segments { - if segment.ID != segmentID { - continue - } - source = segment.Text - if i > 0 { - neighbors = append(neighbors, neighboringSegment("preceding", segments[i-1])) - } - if i+1 < len(segments) { - neighbors = append(neighbors, neighboringSegment("following", segments[i+1])) - } - break - } + source := segments[position].Text + if position > 0 { + neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1])) } + if position+1 < len(segments) { + neighbors = append(neighbors, neighboringSegment("following", segments[position+1])) + } + texts := []string{source, corrected} for _, neighbor := range neighbors { texts = append(texts, neighbor.Text) } var entries []schema.GlossaryEntry - if req.Glossary != nil { - vocabulary := NewProtectedVocabulary(req.Glossary) + if glossary != nil { mentioned := make(map[int]bool) for _, text := range texts { for identity := range vocabulary.occurrencesByIdentity(text) { mentioned[identity] = true } } - for i, entry := range req.Glossary.Entries { + for i, entry := range glossary.Entries { if mentioned[i] { entries = append(entries, entry) } diff --git a/internal/framework/validators/llm_context_test.go b/internal/framework/validators/llm_context_test.go index fe7ed25..ed408a3 100644 --- a/internal/framework/validators/llm_context_test.go +++ b/internal/framework/validators/llm_context_test.go @@ -76,3 +76,19 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) { t.Fatalf("context must count toward batch budget: %+v %v", batches, err) } } + +func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) { + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")}) + req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}} + req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Actors"}, {Name: "Jesters"}}} + payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req) + if len(immediate) != 0 || len(payload.Items) != 2 { + t.Fatalf("payload: %+v immediate: %+v", payload, immediate) + } + for i, name := range []string{"Jesters", "Actors"} { + entries := payload.Items[i].GlossaryEntries + if len(entries) != 1 || entries[0].Name != name { + t.Fatalf("candidate %d context: %+v", i, entries) + } + } +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 19078c0..1d1fb26 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -80,6 +80,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator) operations := make([]outcomes.Operation, 0) + llmDecisions := make([]Decision, 0) + partialResult := func() Result { + decisions := append([]Decision(nil), immediate...) + decisions = append(decisions, oversized...) + decisions = append(decisions, llmDecisions...) + return Result{ValidatorName: v.name, Decisions: decisions, Operations: operations} + } sectionIndex := 0 if req.SectionIndex != nil { sectionIndex = *req.SectionIndex @@ -104,24 +111,23 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } itemsForBatching := filterItemsByDecision(validationReq.Items, oversized) if len(itemsForBatching) == 0 { - all := append([]Decision(nil), immediate...) - all = append(all, oversized...) + result := partialResult() + all := result.Decisions if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { - return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err + return result, err } sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) - return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil + return result, nil } batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator) if err != nil { - return Result{ValidatorName: v.name, Decisions: append(append([]Decision(nil), immediate...), oversized...), Operations: operations}, err + return partialResult(), err } - llmDecisions := make([]Decision, 0) for _, batch := range batches { if err := ctx.Err(); err != nil { - return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err + return partialResult(), err } transcriptDescription := "" if req.Config != nil { @@ -129,7 +135,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } variables, err := BuildValidationVariables(batch.Items, transcriptDescription) if err != nil { - return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err + return partialResult(), err } var response LLMValidationResponse @@ -191,7 +197,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...) continue } - return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) + return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err) } reconciled := reconcileLLMResponse(batch.Items, response) @@ -201,12 +207,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, operation.ProposalIndices = reconciled.UnavailableIndices } else { operation.Succeeded = true - if completion.Execution.StructuralRepairs > 0 { - operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired) - } - if completion.Execution.TransientRetries > 0 { - operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered) - } + operation.RecoveryReasons = outcomes.ExecutionRecoveries(completion.Execution) if len(reconciled.IgnoredIndices) > 0 { operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryValidationDecisionsFiltered) } @@ -218,14 +219,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, llmDecisions = append(llmDecisions, reconciled.Decisions...) } - all := append([]Decision(nil), immediate...) - all = append(all, oversized...) - all = append(all, llmDecisions...) + result := partialResult() + all := result.Decisions if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { - return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err + return result, err } sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) - return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil + return result, nil } func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata { @@ -251,28 +251,31 @@ func errPayload(err error) any { func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) { items := make([]LLMValidationItem, 0, len(req.CandidateProposal)) immediate := make([]Decision, 0) - segments := make(map[int]schema.Segment) + var segments []schema.Segment + positions := make(map[int]int) + vocabulary := NewProtectedVocabulary(req.Glossary) if req.WorkingTranscript != nil { - segments = make(map[int]schema.Segment, len(req.WorkingTranscript.Segments)) - for _, seg := range req.WorkingTranscript.Segments { - segments[seg.ID] = seg + segments = req.WorkingTranscript.Segments + for i, seg := range segments { + positions[seg.ID] = i } } for _, p := range req.CandidateProposal { - seg, ok := segments[p.TargetSegmentID] + position, ok := positions[p.TargetSegmentID] if !ok { immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found")) continue } + seg := segments[position] preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy) if !preview.Applicable { immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation")) continue } - neighbors, glossaryEntries := validationContext(req, seg.ID, preview.CorrectedSegmentText) + neighbors, glossaryEntries := validationContext(segments, position, req.Glossary, vocabulary, preview.CorrectedSegmentText) items = append(items, LLMValidationItem{ CorrectionIndex: p.ProposalIndex, SegmentID: p.TargetSegmentID, diff --git a/internal/prompts/assets/prompts/modules.glossary.proposal.yaml b/internal/prompts/assets/prompts/modules.glossary.proposal.yaml index 4b076de..3afb08b 100644 --- a/internal/prompts/assets/prompts/modules.glossary.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.glossary.proposal.yaml @@ -19,15 +19,11 @@ messages: - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - If a segment includes categories, treat them as additional transcript context. - Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary. - - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target. - - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement. - - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - - Do not return corrections where original_text and corrected_text are identical. - - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. + {{ .ProposalContract }} + {{ .HardeningPolicy }} {{ .TranscriptDescriptionBlock }}Glossary: diff --git a/internal/prompts/assets/prompts/modules.grammar.proposal.yaml b/internal/prompts/assets/prompts/modules.grammar.proposal.yaml index c59a6ea..44e01cf 100644 --- a/internal/prompts/assets/prompts/modules.grammar.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.grammar.proposal.yaml @@ -20,17 +20,13 @@ messages: - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - If a segment includes categories, treat them as additional transcript context. - - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target. - - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Choose an original_text span that appears exactly once in the current segment text. - - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed. - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - - Do not return corrections where original_text and corrected_text are identical. - - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. + {{ .ProposalContract }} + {{ .HardeningPolicy }} {{ .TranscriptDescriptionBlock }}Protected glossary/context: diff --git a/internal/prompts/assets/prompts/modules.homophones.proposal.yaml b/internal/prompts/assets/prompts/modules.homophones.proposal.yaml index d32cdc6..505b1f9 100644 --- a/internal/prompts/assets/prompts/modules.homophones.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.homophones.proposal.yaml @@ -19,15 +19,11 @@ messages: - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - If a segment includes categories, treat them as additional transcript context. - - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target. - - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Choose an original_text span that appears exactly once in the current segment text. - - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - - Do not return corrections where original_text and corrected_text are identical. - - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. + {{ .ProposalContract }} + {{ .HardeningPolicy }} {{ .TranscriptDescriptionBlock }}Protected glossary/context: diff --git a/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml b/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml index d56ef47..bf4647f 100644 --- a/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml +++ b/internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml @@ -22,17 +22,13 @@ messages: - Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript. - Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual. - If a segment includes categories, treat them as additional transcript context. - - Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target. - - Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. - Choose an original_text span that appears exactly once in the current segment text. - - corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. - An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result. - Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank. - - Do not return corrections where original_text and corrected_text are identical. - - Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. - - confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. - Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}. + {{ .ProposalContract }} + {{ .HardeningPolicy }} {{ .TranscriptDescriptionBlock }}Protected glossary/context: diff --git a/internal/prompts/assets/prompts/validators.editorial_review.yaml b/internal/prompts/assets/prompts/validators.editorial_review.yaml index 11de151..0b0b388 100644 --- a/internal/prompts/assets/prompts/validators.editorial_review.yaml +++ b/internal/prompts/assets/prompts/validators.editorial_review.yaml @@ -9,7 +9,7 @@ messages: Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision. Rules: - - Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}. + - Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}. - Approve editorial revisions that preserve the underlying meaning of the segment. - Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies. - Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning. @@ -21,8 +21,8 @@ messages: - Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning. - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. - An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty. - - If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - - confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. + + {{ .ValidationContract }} {{ .HardeningPolicy }} diff --git a/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml b/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml index 113321f..bebb2a5 100644 --- a/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml +++ b/internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml @@ -9,7 +9,7 @@ messages: Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning. Rules: - - Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}. + - Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}. - Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment. - Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning. - Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. @@ -17,8 +17,8 @@ messages: - For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone. - Do not reject a correction merely because the literal written word changes. - Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals. - - If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - - confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. + + {{ .ValidationContract }} {{ .HardeningPolicy }} diff --git a/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml b/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml index 7b4fdfa..369ce19 100644 --- a/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml +++ b/internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml @@ -9,15 +9,15 @@ messages: Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction. Rules: - - Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}. + - Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}. - Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction. - Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn". - Reject unrelated substitutions like changing "Lyra" to "Jesters". - Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical. - For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription. - Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions. - - If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. - - confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. + + {{ .ValidationContract }} {{ .HardeningPolicy }} diff --git a/internal/prompts/assets/shared/proposal_contract.md b/internal/prompts/assets/shared/proposal_contract.md new file mode 100644 index 0000000..6e59d33 --- /dev/null +++ b/internal/prompts/assets/shared/proposal_contract.md @@ -0,0 +1,6 @@ +- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target. +- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text. +- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span. +- Do not return corrections where original_text and corrected_text are identical. +- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal. +- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted. diff --git a/internal/prompts/assets/shared/validation_contract.md b/internal/prompts/assets/shared/validation_contract.md new file mode 100644 index 0000000..a79a4ed --- /dev/null +++ b/internal/prompts/assets/shared/validation_contract.md @@ -0,0 +1,3 @@ +- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. +- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages. +- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict. diff --git a/internal/prompts/registry.go b/internal/prompts/registry.go index 131167b..9f2f07a 100644 --- a/internal/prompts/registry.go +++ b/internal/prompts/registry.go @@ -44,7 +44,7 @@ func (m Metadata) DiagnosticsMap() map[string]any { } } -// Filesystem contains the PromptKit definitions and the shared hardening fragment. +// Filesystem contains the PromptKit definitions and shared instruction fragments. func Filesystem() fs.FS { return embeddedAssets } func LookupMetadata(promptID string) (Metadata, bool) { @@ -75,10 +75,21 @@ func RegisteredMetadata() []Metadata { return out } -func HardeningText() string { - raw, err := embeddedAssets.ReadFile("assets/shared/prompt_hardening.md") +// SharedVariables supplies embedded instructions through the normal PromptKit variables. +func SharedVariables() map[string]string { + return map[string]string{ + "HardeningPolicy": HardeningText(), + "ProposalContract": sharedText("proposal_contract.md"), + "ValidationContract": sharedText("validation_contract.md"), + } +} + +func HardeningText() string { return sharedText("prompt_hardening.md") } + +func sharedText(name string) string { + raw, err := embeddedAssets.ReadFile("assets/shared/" + name) if err != nil { - panic(fmt.Sprintf("read embedded hardening policy: %v", err)) + panic(fmt.Sprintf("read embedded prompt fragment %s: %v", name, err)) } return string(raw) } diff --git a/internal/prompts/registry_test.go b/internal/prompts/registry_test.go index 9352812..e101563 100644 --- a/internal/prompts/registry_test.go +++ b/internal/prompts/registry_test.go @@ -25,6 +25,9 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) { "SectionJSON": "{\"text\":\"fixture-section\"}", "PayloadJSON": "[{\"text\":\"fixture-correction\"}]", } + for key, value := range SharedVariables() { + vars[key] = value + } for _, tc := range []struct{ id, instruction, schema, payload string }{ {PromptIDModuleGlossaryProposal, "strongly supported by the glossary", "correction_set.v1.json", "fixture-section"}, {PromptIDModuleHomophonesProposal, "homophone correction assistant", "correction_set.v1.json", "fixture-section"}, @@ -54,6 +57,24 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) { if strings.HasPrefix(tc.id, "modules.") && !strings.Contains(rendered, "fixture-glossary") { t.Error("missing glossary context") } + contract := "ValidationContract" + if strings.HasPrefix(tc.id, "modules.") { + contract = "ProposalContract" + } + for _, line := range strings.Split(strings.TrimSpace(vars[contract]), "\n") { + if strings.Count(rendered, line) != 1 { + t.Errorf("shared instruction missing or duplicated: %q", line) + } + } + for id, guidance := range map[string]string{ + PromptIDModuleGlossaryProposal: "every literal occurrence", + PromptIDModuleSpokenWordProposal: "Accidental repetition of a protected name may be reduced", + PromptIDModuleGrammarProposal: "whole-word article", + } { + if tc.id == id && !strings.Contains(rendered, guidance) { + t.Errorf("missing module guidance: %s", guidance) + } + } missing := map[string]string{"HardeningPolicy": HardeningText()} if _, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, PromptVersion: VersionV1, ProfileID: "fixture", Vars: missing}); err == nil { t.Error("missing variables accepted")