Share prompt instructions and simplify validation helpers
This commit is contained in:
@@ -98,7 +98,9 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
|
||||
for key, value := range req.Variables {
|
||||
vars[key] = value
|
||||
}
|
||||
vars["HardeningPolicy"] = prompts.HardeningText()
|
||||
for key, value := range prompts.SharedVariables() {
|
||||
vars[key] = value
|
||||
}
|
||||
apiKey := ""
|
||||
if backend.Credential != nil {
|
||||
apiKey = backend.Credential.Value()
|
||||
|
||||
@@ -92,3 +92,16 @@ const (
|
||||
WorkCompleted WorkState = "completed"
|
||||
WorkInterrupted WorkState = "interrupted"
|
||||
)
|
||||
|
||||
// ExecutionRecoveries classifies execution counts; callers decide whether their
|
||||
// operation succeeded before recording these reasons.
|
||||
func ExecutionRecoveries(counts ExecutionCounts) []RecoveryReason {
|
||||
var reasons []RecoveryReason
|
||||
if counts.StructuralRepairs > 0 {
|
||||
reasons = append(reasons, RecoveryOutputRepaired)
|
||||
}
|
||||
if counts.TransientRetries > 0 {
|
||||
reasons = append(reasons, RecoveryServiceRecovered)
|
||||
}
|
||||
return reasons
|
||||
}
|
||||
|
||||
@@ -176,12 +176,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
return result, fmt.Errorf("proposal generation completion failed: %w", callErr)
|
||||
}
|
||||
operation.Succeeded = true
|
||||
if completion.Execution.StructuralRepairs > 0 {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
|
||||
}
|
||||
if completion.Execution.TransientRetries > 0 {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
|
||||
}
|
||||
operation.RecoveryReasons = outcomes.ExecutionRecoveries(completion.Execution)
|
||||
|
||||
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
|
||||
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
|
||||
|
||||
@@ -8,39 +8,29 @@ import (
|
||||
// validationContext supplies adjacent source segments within the current chunk
|
||||
// and glossary definitions mentioned in the candidate or that local context.
|
||||
// These fields travel in each item so batching accounts for their size.
|
||||
func validationContext(req Request, segmentID int, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
|
||||
func validationContext(segments []schema.Segment, position int, glossary *schema.Glossary, vocabulary ProtectedVocabulary, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
|
||||
var neighbors []NeighboringSegment
|
||||
var source string
|
||||
if req.WorkingTranscript != nil {
|
||||
segments := req.WorkingTranscript.Segments
|
||||
for i, segment := range segments {
|
||||
if segment.ID != segmentID {
|
||||
continue
|
||||
}
|
||||
source = segment.Text
|
||||
if i > 0 {
|
||||
neighbors = append(neighbors, neighboringSegment("preceding", segments[i-1]))
|
||||
}
|
||||
if i+1 < len(segments) {
|
||||
neighbors = append(neighbors, neighboringSegment("following", segments[i+1]))
|
||||
}
|
||||
break
|
||||
}
|
||||
source := segments[position].Text
|
||||
if position > 0 {
|
||||
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
|
||||
}
|
||||
if position+1 < len(segments) {
|
||||
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
|
||||
}
|
||||
|
||||
texts := []string{source, corrected}
|
||||
for _, neighbor := range neighbors {
|
||||
texts = append(texts, neighbor.Text)
|
||||
}
|
||||
var entries []schema.GlossaryEntry
|
||||
if req.Glossary != nil {
|
||||
vocabulary := NewProtectedVocabulary(req.Glossary)
|
||||
if glossary != nil {
|
||||
mentioned := make(map[int]bool)
|
||||
for _, text := range texts {
|
||||
for identity := range vocabulary.occurrencesByIdentity(text) {
|
||||
mentioned[identity] = true
|
||||
}
|
||||
}
|
||||
for i, entry := range req.Glossary.Entries {
|
||||
for i, entry := range glossary.Entries {
|
||||
if mentioned[i] {
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
@@ -76,3 +76,19 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
|
||||
t.Fatalf("context must count toward batch budget: %+v %v", batches, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
|
||||
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
|
||||
req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Actors"}, {Name: "Jesters"}}}
|
||||
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
|
||||
if len(immediate) != 0 || len(payload.Items) != 2 {
|
||||
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
|
||||
}
|
||||
for i, name := range []string{"Jesters", "Actors"} {
|
||||
entries := payload.Items[i].GlossaryEntries
|
||||
if len(entries) != 1 || entries[0].Name != name {
|
||||
t.Fatalf("candidate %d context: %+v", i, entries)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,6 +80,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
|
||||
operations := make([]outcomes.Operation, 0)
|
||||
llmDecisions := make([]Decision, 0)
|
||||
partialResult := func() Result {
|
||||
decisions := append([]Decision(nil), immediate...)
|
||||
decisions = append(decisions, oversized...)
|
||||
decisions = append(decisions, llmDecisions...)
|
||||
return Result{ValidatorName: v.name, Decisions: decisions, Operations: operations}
|
||||
}
|
||||
sectionIndex := 0
|
||||
if req.SectionIndex != nil {
|
||||
sectionIndex = *req.SectionIndex
|
||||
@@ -104,24 +111,23 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
|
||||
if len(itemsForBatching) == 0 {
|
||||
all := append([]Decision(nil), immediate...)
|
||||
all = append(all, oversized...)
|
||||
result := partialResult()
|
||||
all := result.Decisions
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
|
||||
return result, err
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
|
||||
if err != nil {
|
||||
return Result{ValidatorName: v.name, Decisions: append(append([]Decision(nil), immediate...), oversized...), Operations: operations}, err
|
||||
return partialResult(), err
|
||||
}
|
||||
|
||||
llmDecisions := make([]Decision, 0)
|
||||
for _, batch := range batches {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
|
||||
return partialResult(), err
|
||||
}
|
||||
transcriptDescription := ""
|
||||
if req.Config != nil {
|
||||
@@ -129,7 +135,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
|
||||
if err != nil {
|
||||
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
|
||||
return partialResult(), err
|
||||
}
|
||||
|
||||
var response LLMValidationResponse
|
||||
@@ -191,7 +197,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...)
|
||||
continue
|
||||
}
|
||||
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
||||
return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
||||
}
|
||||
|
||||
reconciled := reconcileLLMResponse(batch.Items, response)
|
||||
@@ -201,12 +207,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
operation.ProposalIndices = reconciled.UnavailableIndices
|
||||
} else {
|
||||
operation.Succeeded = true
|
||||
if completion.Execution.StructuralRepairs > 0 {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
|
||||
}
|
||||
if completion.Execution.TransientRetries > 0 {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
|
||||
}
|
||||
operation.RecoveryReasons = outcomes.ExecutionRecoveries(completion.Execution)
|
||||
if len(reconciled.IgnoredIndices) > 0 {
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryValidationDecisionsFiltered)
|
||||
}
|
||||
@@ -218,14 +219,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
llmDecisions = append(llmDecisions, reconciled.Decisions...)
|
||||
}
|
||||
|
||||
all := append([]Decision(nil), immediate...)
|
||||
all = append(all, oversized...)
|
||||
all = append(all, llmDecisions...)
|
||||
result := partialResult()
|
||||
all := result.Decisions
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
|
||||
return result, err
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
|
||||
@@ -251,28 +251,31 @@ func errPayload(err error) any {
|
||||
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
|
||||
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
|
||||
immediate := make([]Decision, 0)
|
||||
segments := make(map[int]schema.Segment)
|
||||
var segments []schema.Segment
|
||||
positions := make(map[int]int)
|
||||
vocabulary := NewProtectedVocabulary(req.Glossary)
|
||||
if req.WorkingTranscript != nil {
|
||||
segments = make(map[int]schema.Segment, len(req.WorkingTranscript.Segments))
|
||||
for _, seg := range req.WorkingTranscript.Segments {
|
||||
segments[seg.ID] = seg
|
||||
segments = req.WorkingTranscript.Segments
|
||||
for i, seg := range segments {
|
||||
positions[seg.ID] = i
|
||||
}
|
||||
}
|
||||
|
||||
for _, p := range req.CandidateProposal {
|
||||
seg, ok := segments[p.TargetSegmentID]
|
||||
position, ok := positions[p.TargetSegmentID]
|
||||
if !ok {
|
||||
immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found"))
|
||||
continue
|
||||
}
|
||||
|
||||
seg := segments[position]
|
||||
preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy)
|
||||
if !preview.Applicable {
|
||||
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
|
||||
continue
|
||||
}
|
||||
|
||||
neighbors, glossaryEntries := validationContext(req, seg.ID, preview.CorrectedSegmentText)
|
||||
neighbors, glossaryEntries := validationContext(segments, position, req.Glossary, vocabulary, preview.CorrectedSegmentText)
|
||||
items = append(items, LLMValidationItem{
|
||||
CorrectionIndex: p.ProposalIndex,
|
||||
SegmentID: p.TargetSegmentID,
|
||||
|
||||
@@ -19,15 +19,11 @@ messages:
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.
|
||||
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
|
||||
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
|
||||
- Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement.
|
||||
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
|
||||
- Do not return corrections where original_text and corrected_text are identical.
|
||||
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
|
||||
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
|
||||
{{ .ProposalContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Glossary:
|
||||
|
||||
@@ -20,17 +20,13 @@ messages:
|
||||
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
|
||||
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
|
||||
- Choose an original_text span that appears exactly once in the current segment text.
|
||||
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
|
||||
- A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed.
|
||||
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
|
||||
- Do not return corrections where original_text and corrected_text are identical.
|
||||
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
|
||||
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
|
||||
{{ .ProposalContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Protected glossary/context:
|
||||
|
||||
@@ -19,15 +19,11 @@ messages:
|
||||
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
|
||||
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
|
||||
- Choose an original_text span that appears exactly once in the current segment text.
|
||||
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
|
||||
- Do not return corrections where original_text and corrected_text are identical.
|
||||
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
|
||||
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
|
||||
{{ .ProposalContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Protected glossary/context:
|
||||
|
||||
@@ -22,17 +22,13 @@ messages:
|
||||
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
|
||||
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
|
||||
- If a segment includes categories, treat them as additional transcript context.
|
||||
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
|
||||
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
|
||||
- Choose an original_text span that appears exactly once in the current segment text.
|
||||
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
|
||||
- An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result.
|
||||
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
|
||||
- Do not return corrections where original_text and corrected_text are identical.
|
||||
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
|
||||
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
|
||||
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
|
||||
|
||||
{{ .ProposalContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
{{ .TranscriptDescriptionBlock }}Protected glossary/context:
|
||||
|
||||
@@ -9,7 +9,7 @@ messages:
|
||||
Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.
|
||||
|
||||
Rules:
|
||||
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
|
||||
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
|
||||
- Approve editorial revisions that preserve the underlying meaning of the segment.
|
||||
- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.
|
||||
- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.
|
||||
@@ -21,8 +21,8 @@ messages:
|
||||
- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.
|
||||
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
|
||||
- An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty.
|
||||
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
|
||||
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
|
||||
|
||||
{{ .ValidationContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ messages:
|
||||
Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.
|
||||
|
||||
Rules:
|
||||
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
|
||||
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
|
||||
- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.
|
||||
- Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning.
|
||||
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
|
||||
@@ -17,8 +17,8 @@ messages:
|
||||
- For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone.
|
||||
- Do not reject a correction merely because the literal written word changes.
|
||||
- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.
|
||||
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
|
||||
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
|
||||
|
||||
{{ .ValidationContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
|
||||
@@ -9,15 +9,15 @@ messages:
|
||||
Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.
|
||||
|
||||
Rules:
|
||||
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
|
||||
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
|
||||
- Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.
|
||||
- Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn".
|
||||
- Reject unrelated substitutions like changing "Lyra" to "Jesters".
|
||||
- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.
|
||||
- For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription.
|
||||
- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.
|
||||
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
|
||||
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
|
||||
|
||||
{{ .ValidationContract }}
|
||||
|
||||
{{ .HardeningPolicy }}
|
||||
|
||||
|
||||
6
internal/prompts/assets/shared/proposal_contract.md
Normal file
6
internal/prompts/assets/shared/proposal_contract.md
Normal file
@@ -0,0 +1,6 @@
|
||||
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
|
||||
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
|
||||
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
|
||||
- Do not return corrections where original_text and corrected_text are identical.
|
||||
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
|
||||
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
|
||||
3
internal/prompts/assets/shared/validation_contract.md
Normal file
3
internal/prompts/assets/shared/validation_contract.md
Normal file
@@ -0,0 +1,3 @@
|
||||
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema.
|
||||
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
|
||||
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
|
||||
@@ -44,7 +44,7 @@ func (m Metadata) DiagnosticsMap() map[string]any {
|
||||
}
|
||||
}
|
||||
|
||||
// Filesystem contains the PromptKit definitions and the shared hardening fragment.
|
||||
// Filesystem contains the PromptKit definitions and shared instruction fragments.
|
||||
func Filesystem() fs.FS { return embeddedAssets }
|
||||
|
||||
func LookupMetadata(promptID string) (Metadata, bool) {
|
||||
@@ -75,10 +75,21 @@ func RegisteredMetadata() []Metadata {
|
||||
return out
|
||||
}
|
||||
|
||||
func HardeningText() string {
|
||||
raw, err := embeddedAssets.ReadFile("assets/shared/prompt_hardening.md")
|
||||
// SharedVariables supplies embedded instructions through the normal PromptKit variables.
|
||||
func SharedVariables() map[string]string {
|
||||
return map[string]string{
|
||||
"HardeningPolicy": HardeningText(),
|
||||
"ProposalContract": sharedText("proposal_contract.md"),
|
||||
"ValidationContract": sharedText("validation_contract.md"),
|
||||
}
|
||||
}
|
||||
|
||||
func HardeningText() string { return sharedText("prompt_hardening.md") }
|
||||
|
||||
func sharedText(name string) string {
|
||||
raw, err := embeddedAssets.ReadFile("assets/shared/" + name)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("read embedded hardening policy: %v", err))
|
||||
panic(fmt.Sprintf("read embedded prompt fragment %s: %v", name, err))
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
@@ -25,6 +25,9 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
|
||||
"SectionJSON": "{\"text\":\"fixture-section\"}",
|
||||
"PayloadJSON": "[{\"text\":\"fixture-correction\"}]",
|
||||
}
|
||||
for key, value := range SharedVariables() {
|
||||
vars[key] = value
|
||||
}
|
||||
for _, tc := range []struct{ id, instruction, schema, payload string }{
|
||||
{PromptIDModuleGlossaryProposal, "strongly supported by the glossary", "correction_set.v1.json", "fixture-section"},
|
||||
{PromptIDModuleHomophonesProposal, "homophone correction assistant", "correction_set.v1.json", "fixture-section"},
|
||||
@@ -54,6 +57,24 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
|
||||
if strings.HasPrefix(tc.id, "modules.") && !strings.Contains(rendered, "fixture-glossary") {
|
||||
t.Error("missing glossary context")
|
||||
}
|
||||
contract := "ValidationContract"
|
||||
if strings.HasPrefix(tc.id, "modules.") {
|
||||
contract = "ProposalContract"
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimSpace(vars[contract]), "\n") {
|
||||
if strings.Count(rendered, line) != 1 {
|
||||
t.Errorf("shared instruction missing or duplicated: %q", line)
|
||||
}
|
||||
}
|
||||
for id, guidance := range map[string]string{
|
||||
PromptIDModuleGlossaryProposal: "every literal occurrence",
|
||||
PromptIDModuleSpokenWordProposal: "Accidental repetition of a protected name may be reduced",
|
||||
PromptIDModuleGrammarProposal: "whole-word article",
|
||||
} {
|
||||
if tc.id == id && !strings.Contains(rendered, guidance) {
|
||||
t.Errorf("missing module guidance: %s", guidance)
|
||||
}
|
||||
}
|
||||
missing := map[string]string{"HardeningPolicy": HardeningText()}
|
||||
if _, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, PromptVersion: VersionV1, ProfileID: "fixture", Vars: missing}); err == nil {
|
||||
t.Error("missing variables accepted")
|
||||
|
||||
Reference in New Issue
Block a user