Share prompt instructions and simplify validation helpers

This commit is contained in:
2026-09-25 13:26:51 +00:00
parent 5feec1bf14
commit 07a4b65570
20 changed files with 172 additions and 100 deletions

View File

@@ -98,7 +98,9 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
for key, value := range req.Variables {
vars[key] = value
}
vars["HardeningPolicy"] = prompts.HardeningText()
for key, value := range prompts.SharedVariables() {
vars[key] = value
}
apiKey := ""
if backend.Credential != nil {
apiKey = backend.Credential.Value()

View File

@@ -92,3 +92,16 @@ const (
WorkCompleted WorkState = "completed"
WorkInterrupted WorkState = "interrupted"
)
// ExecutionRecoveries classifies execution counts; callers decide whether their
// operation succeeded before recording these reasons.
func ExecutionRecoveries(counts ExecutionCounts) []RecoveryReason {
var reasons []RecoveryReason
if counts.StructuralRepairs > 0 {
reasons = append(reasons, RecoveryOutputRepaired)
}
if counts.TransientRetries > 0 {
reasons = append(reasons, RecoveryServiceRecovered)
}
return reasons
}

View File

@@ -176,12 +176,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
return result, fmt.Errorf("proposal generation completion failed: %w", callErr)
}
operation.Succeeded = true
if completion.Execution.StructuralRepairs > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
}
if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
}
operation.RecoveryReasons = outcomes.ExecutionRecoveries(completion.Execution)
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))

View File

@@ -8,39 +8,29 @@ import (
// validationContext supplies adjacent source segments within the current chunk
// and glossary definitions mentioned in the candidate or that local context.
// These fields travel in each item so batching accounts for their size.
func validationContext(req Request, segmentID int, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
func validationContext(segments []schema.Segment, position int, glossary *schema.Glossary, vocabulary ProtectedVocabulary, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
var neighbors []NeighboringSegment
var source string
if req.WorkingTranscript != nil {
segments := req.WorkingTranscript.Segments
for i, segment := range segments {
if segment.ID != segmentID {
continue
}
source = segment.Text
if i > 0 {
neighbors = append(neighbors, neighboringSegment("preceding", segments[i-1]))
}
if i+1 < len(segments) {
neighbors = append(neighbors, neighboringSegment("following", segments[i+1]))
}
break
}
source := segments[position].Text
if position > 0 {
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
}
if position+1 < len(segments) {
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
}
texts := []string{source, corrected}
for _, neighbor := range neighbors {
texts = append(texts, neighbor.Text)
}
var entries []schema.GlossaryEntry
if req.Glossary != nil {
vocabulary := NewProtectedVocabulary(req.Glossary)
if glossary != nil {
mentioned := make(map[int]bool)
for _, text := range texts {
for identity := range vocabulary.occurrencesByIdentity(text) {
mentioned[identity] = true
}
}
for i, entry := range req.Glossary.Entries {
for i, entry := range glossary.Entries {
if mentioned[i] {
entries = append(entries, entry)
}

View File

@@ -76,3 +76,19 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
t.Fatalf("context must count toward batch budget: %+v %v", batches, err)
}
}
func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Actors"}, {Name: "Jesters"}}}
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if len(immediate) != 0 || len(payload.Items) != 2 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
for i, name := range []string{"Jesters", "Actors"} {
entries := payload.Items[i].GlossaryEntries
if len(entries) != 1 || entries[0].Name != name {
t.Fatalf("candidate %d context: %+v", i, entries)
}
}
}

View File

@@ -80,6 +80,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
operations := make([]outcomes.Operation, 0)
llmDecisions := make([]Decision, 0)
partialResult := func() Result {
decisions := append([]Decision(nil), immediate...)
decisions = append(decisions, oversized...)
decisions = append(decisions, llmDecisions...)
return Result{ValidatorName: v.name, Decisions: decisions, Operations: operations}
}
sectionIndex := 0
if req.SectionIndex != nil {
sectionIndex = *req.SectionIndex
@@ -104,24 +111,23 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
if len(itemsForBatching) == 0 {
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
result := partialResult()
all := result.Decisions
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
return result, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
return result, nil
}
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
if err != nil {
return Result{ValidatorName: v.name, Decisions: append(append([]Decision(nil), immediate...), oversized...), Operations: operations}, err
return partialResult(), err
}
llmDecisions := make([]Decision, 0)
for _, batch := range batches {
if err := ctx.Err(); err != nil {
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
return partialResult(), err
}
transcriptDescription := ""
if req.Config != nil {
@@ -129,7 +135,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
if err != nil {
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, err
return partialResult(), err
}
var response LLMValidationResponse
@@ -191,7 +197,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...)
continue
}
return Result{ValidatorName: v.name, Decisions: append(append(append([]Decision(nil), immediate...), oversized...), llmDecisions...), Operations: operations}, fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
}
reconciled := reconcileLLMResponse(batch.Items, response)
@@ -201,12 +207,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
operation.ProposalIndices = reconciled.UnavailableIndices
} else {
operation.Succeeded = true
if completion.Execution.StructuralRepairs > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryOutputRepaired)
}
if completion.Execution.TransientRetries > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryServiceRecovered)
}
operation.RecoveryReasons = outcomes.ExecutionRecoveries(completion.Execution)
if len(reconciled.IgnoredIndices) > 0 {
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryValidationDecisionsFiltered)
}
@@ -218,14 +219,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
llmDecisions = append(llmDecisions, reconciled.Decisions...)
}
all := append([]Decision(nil), immediate...)
all = append(all, oversized...)
all = append(all, llmDecisions...)
result := partialResult()
all := result.Decisions
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, err
return result, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return Result{ValidatorName: v.name, Decisions: all, Operations: operations}, nil
return result, nil
}
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
@@ -251,28 +251,31 @@ func errPayload(err error) any {
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
immediate := make([]Decision, 0)
segments := make(map[int]schema.Segment)
var segments []schema.Segment
positions := make(map[int]int)
vocabulary := NewProtectedVocabulary(req.Glossary)
if req.WorkingTranscript != nil {
segments = make(map[int]schema.Segment, len(req.WorkingTranscript.Segments))
for _, seg := range req.WorkingTranscript.Segments {
segments[seg.ID] = seg
segments = req.WorkingTranscript.Segments
for i, seg := range segments {
positions[seg.ID] = i
}
}
for _, p := range req.CandidateProposal {
seg, ok := segments[p.TargetSegmentID]
position, ok := positions[p.TargetSegmentID]
if !ok {
immediate = append(immediate, rejection(p.ProposalIndex, ReasonMissingTargetSegment, "target segment was not found"))
continue
}
seg := segments[position]
preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy)
if !preview.Applicable {
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
continue
}
neighbors, glossaryEntries := validationContext(req, seg.ID, preview.CorrectedSegmentText)
neighbors, glossaryEntries := validationContext(segments, position, req.Glossary, vocabulary, preview.CorrectedSegmentText)
items = append(items, LLMValidationItem{
CorrectionIndex: p.ProposalIndex,
SegmentID: p.TargetSegmentID,

View File

@@ -19,15 +19,11 @@ messages:
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .ProposalContract }}
{{ .HardeningPolicy }}
{{ .TranscriptDescriptionBlock }}Glossary:

View File

@@ -20,17 +20,13 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed.
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .ProposalContract }}
{{ .HardeningPolicy }}
{{ .TranscriptDescriptionBlock }}Protected glossary/context:

View File

@@ -19,15 +19,11 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .ProposalContract }}
{{ .HardeningPolicy }}
{{ .TranscriptDescriptionBlock }}Protected glossary/context:

View File

@@ -22,17 +22,13 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result.
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .ProposalContract }}
{{ .HardeningPolicy }}
{{ .TranscriptDescriptionBlock }}Protected glossary/context:

View File

@@ -9,7 +9,7 @@ messages:
Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.
Rules:
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
- Approve editorial revisions that preserve the underlying meaning of the segment.
- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.
- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.
@@ -21,8 +21,8 @@ messages:
- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
- An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .ValidationContract }}
{{ .HardeningPolicy }}

View File

@@ -9,7 +9,7 @@ messages:
Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.
Rules:
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.
- Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning.
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
@@ -17,8 +17,8 @@ messages:
- For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone.
- Do not reject a correction merely because the literal written word changes.
- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .ValidationContract }}
{{ .HardeningPolicy }}

View File

@@ -9,15 +9,15 @@ messages:
Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.
Rules:
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
- Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.
- Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn".
- Reject unrelated substitutions like changing "Lyra" to "Jesters".
- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.
- For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription.
- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .ValidationContract }}
{{ .HardeningPolicy }}

View File

@@ -0,0 +1,6 @@
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.

View File

@@ -0,0 +1,3 @@
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.

View File

@@ -44,7 +44,7 @@ func (m Metadata) DiagnosticsMap() map[string]any {
}
}
// Filesystem contains the PromptKit definitions and the shared hardening fragment.
// Filesystem contains the PromptKit definitions and shared instruction fragments.
func Filesystem() fs.FS { return embeddedAssets }
func LookupMetadata(promptID string) (Metadata, bool) {
@@ -75,10 +75,21 @@ func RegisteredMetadata() []Metadata {
return out
}
func HardeningText() string {
raw, err := embeddedAssets.ReadFile("assets/shared/prompt_hardening.md")
// SharedVariables supplies embedded instructions through the normal PromptKit variables.
func SharedVariables() map[string]string {
return map[string]string{
"HardeningPolicy": HardeningText(),
"ProposalContract": sharedText("proposal_contract.md"),
"ValidationContract": sharedText("validation_contract.md"),
}
}
func HardeningText() string { return sharedText("prompt_hardening.md") }
func sharedText(name string) string {
raw, err := embeddedAssets.ReadFile("assets/shared/" + name)
if err != nil {
panic(fmt.Sprintf("read embedded hardening policy: %v", err))
panic(fmt.Sprintf("read embedded prompt fragment %s: %v", name, err))
}
return string(raw)
}

View File

@@ -25,6 +25,9 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
"SectionJSON": "{\"text\":\"fixture-section\"}",
"PayloadJSON": "[{\"text\":\"fixture-correction\"}]",
}
for key, value := range SharedVariables() {
vars[key] = value
}
for _, tc := range []struct{ id, instruction, schema, payload string }{
{PromptIDModuleGlossaryProposal, "strongly supported by the glossary", "correction_set.v1.json", "fixture-section"},
{PromptIDModuleHomophonesProposal, "homophone correction assistant", "correction_set.v1.json", "fixture-section"},
@@ -54,6 +57,24 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
if strings.HasPrefix(tc.id, "modules.") && !strings.Contains(rendered, "fixture-glossary") {
t.Error("missing glossary context")
}
contract := "ValidationContract"
if strings.HasPrefix(tc.id, "modules.") {
contract = "ProposalContract"
}
for _, line := range strings.Split(strings.TrimSpace(vars[contract]), "\n") {
if strings.Count(rendered, line) != 1 {
t.Errorf("shared instruction missing or duplicated: %q", line)
}
}
for id, guidance := range map[string]string{
PromptIDModuleGlossaryProposal: "every literal occurrence",
PromptIDModuleSpokenWordProposal: "Accidental repetition of a protected name may be reduced",
PromptIDModuleGrammarProposal: "whole-word article",
} {
if tc.id == id && !strings.Contains(rendered, guidance) {
t.Errorf("missing module guidance: %s", guidance)
}
}
missing := map[string]string{"HardeningPolicy": HardeningText()}
if _, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, PromptVersion: VersionV1, ProfileID: "fixture", Vars: missing}); err == nil {
t.Error("missing variables accepted")