eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Move prompts into embedded Markdown assets

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 18:49:06 +00:00
parent d6126bf52b
commit 037121e9ce
43 changed files with 940 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
internal/framework/validators/llm_prompt_builders.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
@@ -12,20 +13,13 @@ func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem,
if err != nil {
return nil, err
}
system := "You are Audita, a conservative spoken-form validation assistant. Evaluate whether each proposed correction is plausibly explained by a homophone, phonetic similarity, or a common mistranscription of spoken English. Your job is not to improve style or readability. Approve only when the corrected text is a plausible recovery of the words that were likely spoken."
user := "Review these proposed transcript corrections and decide whether each one is a plausible spoken-form correction.\n\n" +
"Rules:\n" +
"- Return one validation decision for every correction_index in the input.\n" +
"- Approve when the original text and corrected text are plausibly related by homophone confusion, phonetic similarity, or a common spoken-word mistranscription, and the surrounding segment context supports the correction.\n" +
"- Examples that may be approved when context supports them: changing \"gestures\" to \"Jesters\", \"rank\" to \"Hrank\", or \"dam\" to \"damn\".\n" +
"- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n" +
"- Judge spoken-form plausibility, not whether the correction is cleaner, more formal, or more grammatical.\n" +
"- Do not approve paraphrases, stylistic rewrites, or arbitrary semantic substitutions.\n" +
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorSpokenFormPlausibility, map[string]string{
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
"PayloadJSON": payloadJSON,
})
if err != nil {
return nil, err
}
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
@@ -34,20 +28,13 @@ func BuildMeaningReversalMessages(validationPayload []LLMValidationItem, transcr
if err != nil {
return nil, err
}
system := "You are Audita, a narrow semantic-reversal validation assistant. Evaluate whether each proposed correction changes a word to its antonym or otherwise reverses the meaning of the full segment. Do not treat every word substitution as a problem; focus specifically on antonyms and meaning reversals."
user := "Review these proposed transcript corrections and decide whether each one avoids reversing the segment meaning.\n\n" +
"Rules:\n" +
"- Return one validation decision for every correction_index in the input.\n" +
"- Reject corrections that introduce antonyms or otherwise reverse the meaning of the original segment.\n" +
"- Reject examples like changing \"visible\" to \"invisible\" or \"up\" to \"down\" when that reverses the segment meaning.\n" +
"- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" +
"- Do not reject a correction merely because the literal written word changes.\n" +
"- Do not act as a general semantic-style reviewer; this validator is only a guard against antonyms and meaning reversals.\n" +
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorMeaningReversalReview, map[string]string{
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
"PayloadJSON": payloadJSON,
})
if err != nil {
return nil, err
}
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
@@ -56,34 +43,44 @@ func BuildEditorialMessages(validationPayload []LLMValidationItem, transcriptDes
if err != nil {
return nil, err
}
system := "You are Audita, a conservative editorial validation assistant. Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, homophone or mistranscription corrections, and similar low-risk editorial cleanup."
user := "Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.\n\n" +
"Rules:\n" +
"- Return one validation decision for every correction_index in the input.\n" +
"- Approve editorial revisions that preserve the underlying meaning of the segment.\n" +
"- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n" +
"- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.\n" +
"- Approve low-risk homophone or mistranscription corrections when they are contextually well supported.\n" +
"- Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving.\n" +
"- Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n" +
"- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n" +
"- Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart.\n" +
"- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.\n" +
"- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" +
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorEditorialReview, map[string]string{
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
"PayloadJSON": payloadJSON,
})
if err != nil {
return nil, err
}
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildGrammarReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload, transcriptDescription)
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
}
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorGrammarReview, map[string]string{
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
"PayloadJSON": payloadJSON,
})
if err != nil {
return nil, err
}
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildSpokenWordReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload, transcriptDescription)
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
}
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDValidatorSpokenWordReview, map[string]string{
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
"PayloadJSON": payloadJSON,
})
if err != nil {
return nil, err
}
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func marshalPromptPayload(validationPayload []LLMValidationItem) (string, error) {

26
internal/framework/validators/llm_validators.go
View File

@@ -11,6 +11,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
type LLMPromptBuilder func(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error)
@@ -112,12 +113,20 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
stage := fmt.Sprintf("%s:%s:batch-%04d", req.ModuleInstance, v.name, batch.BatchIndex)
promptMetadata := validatorPromptMetadata(v.validatorType)
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"prompt_metadata": map[string]any{
"prompt_id": promptMetadata.PromptID,
"prompt_version": promptMetadata.PromptVersion,
"prompt_source": promptMetadata.PromptSource,
"embedded_path": promptMetadata.EmbeddedPath,
"sha256": promptMetadata.SHA256,
},
"response_schema": map[string]any{
"id": responseSchema.ID,
"version": responseSchema.Version,
@@ -153,6 +162,23 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
return Result{ValidatorName: v.name, Decisions: all}, nil
}
func validatorPromptMetadata(validatorType LLMValidatorType) prompts.Metadata {
switch validatorType {
case LLMValidatorTypeSpokenFormPlausibility:
return prompts.MustLookupMetadata(prompts.PromptIDValidatorSpokenFormPlausibility)
case LLMValidatorTypeMeaningReversal:
return prompts.MustLookupMetadata(prompts.PromptIDValidatorMeaningReversalReview)
case LLMValidatorTypeEditorialReview:
return prompts.MustLookupMetadata(prompts.PromptIDValidatorEditorialReview)
case LLMValidatorTypeGrammarReview:
return prompts.MustLookupMetadata(prompts.PromptIDValidatorGrammarReview)
case LLMValidatorTypeSpokenWordReview:
return prompts.MustLookupMetadata(prompts.PromptIDValidatorSpokenWordReview)
default:
return prompts.Metadata{}
}
}
func errPayload(err error) any {
if err == nil {
return nil

13
internal/framework/validators/llm_validators_test.go
View File

@@ -359,6 +359,19 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
}
promptMap, ok := metadata["prompt_metadata"].(map[string]any)
if !ok {
t.Fatalf("expected prompt_metadata map, got %T", metadata["prompt_metadata"])
}
if promptMap["prompt_id"] != "validators.spoken_form_plausibility" || promptMap["prompt_version"] != "v1" || promptMap["prompt_source"] != "builtin" {
t.Fatalf("unexpected prompt metadata: %v", promptMap)
}
if _, ok := promptMap["embedded_path"].(string); !ok {
t.Fatalf("expected embedded_path in prompt metadata: %v", promptMap)
}
if _, ok := promptMap["sha256"].(string); !ok {
t.Fatalf("expected sha256 in prompt metadata: %v", promptMap)
}
}
func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) {