eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add transcript description prompt context

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 12:13:32 +00:00
parent de99467ede
commit ebbd2c8a63
27 changed files with 542 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
internal/framework/validators/llm_prompt_builders.go
View File

@@ -3,9 +3,11 @@ package validators
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
@@ -22,11 +24,12 @@ func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem)
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildMeaningReversalMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
func BuildMeaningReversalMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
@@ -43,11 +46,12 @@ func BuildMeaningReversalMessages(validationPayload []LLMValidationItem) ([]LLMM
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildEditorialMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
func BuildEditorialMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
@@ -69,16 +73,17 @@ func BuildEditorialMessages(validationPayload []LLMValidationItem) ([]LLMMessage
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildGrammarReviewMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload)
func BuildGrammarReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload, transcriptDescription)
}
func BuildSpokenWordReviewMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload)
func BuildSpokenWordReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload, transcriptDescription)
}
func marshalPromptPayload(validationPayload []LLMValidationItem) (string, error) {

8
internal/framework/validators/llm_validators.go
View File

@@ -13,7 +13,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type LLMPromptBuilder func(validationPayload []LLMValidationItem) ([]LLMMessage, error)
type LLMPromptBuilder func(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error)
type LLMBackedValidator struct {
name string
@@ -84,7 +84,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
llmDecisions := make([]Decision, 0)
for _, batch := range batches {
messages, err := v.promptBuilder(batch.Items)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := v.promptBuilder(batch.Items, transcriptDescription)
if err != nil {
return Result{}, err
}

25
internal/framework/validators/llm_validators_test.go
View File

@@ -167,7 +167,7 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters", Categories: []string{"narration"}}}
tests := []struct {
name string
build func([]LLMValidationItem) ([]LLMMessage, error)
build func([]LLMValidationItem, string) ([]LLMMessage, error)
mustHas []string
}{
{"spoken_form", BuildSpokenFormPlausibilityMessages, []string{"plausible spoken-form", "correction_index", "original_segment_text", "corrected_segment_text"}},
@@ -179,7 +179,7 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
msgs, err := tt.build(payload)
msgs, err := tt.build(payload, "Discussion among party members in a dungeon.")
if err != nil {
t.Fatalf("build err: %v", err)
}
@@ -192,10 +192,31 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
for _, needle := range []string{
"Transcript description (background context only):",
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(combined, needle) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
})
}
}
func TestPromptBuildersOmitTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters"}}
msgs, err := BuildSpokenFormPlausibilityMessages(payload, " ")
if err != nil {
t.Fatalf("build err: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("did not expect empty transcript description section in prompt")
}
}
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},