eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Share prompt instructions and simplify validation helpers

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 13:26:51 +00:00
parent 5feec1bf14
commit 07a4b65570
20 changed files with 172 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/prompts/assets/prompts/modules.glossary.proposal.yaml
View File

@@ -19,15 +19,11 @@ messages:
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Plural forms of glossary names and aliases are allowed targets when spoken similarity and context support them, even if the plural is not explicitly listed in the glossary.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Audita replaces every literal occurrence of original_text within the target segment. If only one occurrence should change, include enough unchanged context to distinguish it and preserve that context in the replacement.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .ProposalContract }}
{{ .HardeningPolicy }}
{{ .TranscriptDescriptionBlock }}Glossary:

8
internal/prompts/assets/prompts/modules.grammar.proposal.yaml
View File

@@ -20,17 +20,13 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- A whitespace-only original_text can identify a spacing change: replacing the two spaces in "hello world" with one space yields "hello world". An empty corrected_text deletes its identified span when punctuation or spacing should be removed.
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "hello world", "corrected_text": "hello world", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .ProposalContract }}
{{ .HardeningPolicy }}
{{ .TranscriptDescriptionBlock }}Protected glossary/context:

8
internal/prompts/assets/prompts/modules.homophones.proposal.yaml
View File

@@ -19,15 +19,11 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .ProposalContract }}
{{ .HardeningPolicy }}
{{ .TranscriptDescriptionBlock }}Protected glossary/context:

8
internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml
View File

@@ -22,17 +22,13 @@ messages:
- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.
- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.
- If a segment includes categories, treat them as additional transcript context.
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- Choose an original_text span that appears exactly once in the current segment text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- An empty corrected_text deletes the identified span. For example, removing "uh " from "uh hello" leaves "hello"; include adjacent whitespace or punctuation in the span when needed for the intended literal result.
- Omit a correction that would leave the segment blank; do not invent padding or content to keep it nonblank.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.
- Follow the supplied JSON schema. Output example: {"corrections": [{"id": 1, "original_text": "uh ", "corrected_text": "", "confidence": 0.95}]}. If no corrections are needed, return {"corrections": []}.
{{ .ProposalContract }}
{{ .HardeningPolicy }}
{{ .TranscriptDescriptionBlock }}Protected glossary/context:

6
internal/prompts/assets/prompts/validators.editorial_review.yaml
View File

@@ -9,7 +9,7 @@ messages:
Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.
Rules:
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit preserves meaning."}]}.
- Approve editorial revisions that preserve the underlying meaning of the segment.
- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.
- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.
@@ -21,8 +21,8 @@ messages:
- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
- An empty corrected_text deletes the identified span; judge the actual before/after segment, including adjacent spacing and punctuation. Do not approve or reject solely because the replacement is empty.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .ValidationContract }}
{{ .HardeningPolicy }}

6
internal/prompts/assets/prompts/validators.meaning_reversal_review.yaml
View File

@@ -9,7 +9,7 @@ messages:
Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.
Rules:
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The edit does not reverse meaning."}]}.
- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.
- Reject examples like changing "visible" to "invisible" or "up" to "down" when that reverses the segment meaning.
- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.
@@ -17,8 +17,8 @@ messages:
- For example, deleting "not " from "The door is not open" to produce "The door is open" reverses meaning. Judge the actual before/after context, not the presence of a particular token alone.
- Do not reject a correction merely because the literal written word changes.
- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .ValidationContract }}
{{ .HardeningPolicy }}

6
internal/prompts/assets/prompts/validators.spoken_form_plausibility.yaml
View File

@@ -9,15 +9,15 @@ messages:
Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.
Rules:
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema. Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
- Output example: {"validations": [{"correction_index": 1, "approved": true, "confidence": 0.95, "reason": "The replacement is a plausible mishearing."}]}.
- Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.
- Examples that may be approved when context supports them: changing "gestures" to "Jesters", "rank" to "Hrank", or "dam" to "damn".
- Reject unrelated substitutions like changing "Lyra" to "Jesters".
- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.
- For every correction, evaluate the full original_segment_text and corrected_segment_text, not only the replacement span. When a span is deleted, an empty corrected_text alone does not establish or rule out a spoken-form mistranscription.
- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.
{{ .ValidationContract }}
{{ .HardeningPolicy }}

6
internal/prompts/assets/shared/proposal_contract.md Normal file
View File

@@ -0,0 +1,6 @@
- Use the segment's local id (1, 2, 3, ...) from this request to identify the correction target.
- Use a nonempty original_text copied exactly from the supplied segment; matching is literal and case-sensitive, including whitespace and punctuation. Include unchanged surrounding text when needed to identify the intended edit, preserving that context in corrected_text.
- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.
- Do not return corrections where original_text and corrected_text are identical.
- Return only proposed corrections. Multiple corrections may use the same segment id; each original_text refers to the supplied segment, not the imagined result of another proposal.
- confidence must be between 0.0 and 1.0 and express confidence that the proposed correction is warranted.

3
internal/prompts/assets/shared/validation_contract.md Normal file
View File

@@ -0,0 +1,3 @@
- Return one decision for each numbered candidate, including rejections. correction_index identifies the candidate within this request (1, 2, 3, ...). Follow the supplied JSON schema.
- If supplied, neighboring_segments identify preceding/following source context, not additional correction requests. glossary_entries provide reference definitions for terms mentioned in the before/after text or those neighbors. Use this evidence and categories where relevant; do not assume access to audio or other transcript passages.
- confidence must be between 0.0 and 1.0 and express confidence in your verdict. A confident rejection may have high confidence; approved, not confidence, states the verdict.

19
internal/prompts/registry.go
View File

@@ -44,7 +44,7 @@ func (m Metadata) DiagnosticsMap() map[string]any {
}
}
// Filesystem contains the PromptKit definitions and the shared hardening fragment.
// Filesystem contains the PromptKit definitions and shared instruction fragments.
func Filesystem() fs.FS { return embeddedAssets }
func LookupMetadata(promptID string) (Metadata, bool) {
@@ -75,10 +75,21 @@ func RegisteredMetadata() []Metadata {
return out
}
func HardeningText() string {
raw, err := embeddedAssets.ReadFile("assets/shared/prompt_hardening.md")
// SharedVariables supplies embedded instructions through the normal PromptKit variables.
func SharedVariables() map[string]string {
return map[string]string{
"HardeningPolicy": HardeningText(),
"ProposalContract": sharedText("proposal_contract.md"),
"ValidationContract": sharedText("validation_contract.md"),
}
}
func HardeningText() string { return sharedText("prompt_hardening.md") }
func sharedText(name string) string {
raw, err := embeddedAssets.ReadFile("assets/shared/" + name)
if err != nil {
panic(fmt.Sprintf("read embedded hardening policy: %v", err))
panic(fmt.Sprintf("read embedded prompt fragment %s: %v", name, err))
}
return string(raw)
}

21
internal/prompts/registry_test.go
View File

@@ -25,6 +25,9 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
"SectionJSON": "{\"text\":\"fixture-section\"}",
"PayloadJSON": "[{\"text\":\"fixture-correction\"}]",
}
for key, value := range SharedVariables() {
vars[key] = value
}
for _, tc := range []struct{ id, instruction, schema, payload string }{
{PromptIDModuleGlossaryProposal, "strongly supported by the glossary", "correction_set.v1.json", "fixture-section"},
{PromptIDModuleHomophonesProposal, "homophone correction assistant", "correction_set.v1.json", "fixture-section"},
@@ -54,6 +57,24 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
if strings.HasPrefix(tc.id, "modules.") && !strings.Contains(rendered, "fixture-glossary") {
t.Error("missing glossary context")
}
contract := "ValidationContract"
if strings.HasPrefix(tc.id, "modules.") {
contract = "ProposalContract"
}
for _, line := range strings.Split(strings.TrimSpace(vars[contract]), "\n") {
if strings.Count(rendered, line) != 1 {
t.Errorf("shared instruction missing or duplicated: %q", line)
}
}
for id, guidance := range map[string]string{
PromptIDModuleGlossaryProposal: "every literal occurrence",
PromptIDModuleSpokenWordProposal: "Accidental repetition of a protected name may be reduced",
PromptIDModuleGrammarProposal: "whole-word article",
} {
if tc.id == id && !strings.Contains(rendered, guidance) {
t.Errorf("missing module guidance: %s", guidance)
}
}
missing := map[string]string{"HardeningPolicy": HardeningText()}
if _, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, PromptVersion: VersionV1, ProfileID: "fixture", Vars: missing}); err == nil {
t.Error("missing variables accepted")