eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add transcript description prompt context

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 12:13:32 +00:00
parent de99467ede
commit ebbd2c8a63
27 changed files with 542 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
internal/framework/validators/llm_validators_test.go
View File

@@ -167,7 +167,7 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters", Categories: []string{"narration"}}}
tests := []struct {
name string
build func([]LLMValidationItem) ([]LLMMessage, error)
build func([]LLMValidationItem, string) ([]LLMMessage, error)
mustHas []string
}{
{"spoken_form", BuildSpokenFormPlausibilityMessages, []string{"plausible spoken-form", "correction_index", "original_segment_text", "corrected_segment_text"}},
@@ -179,7 +179,7 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
msgs, err := tt.build(payload)
msgs, err := tt.build(payload, "Discussion among party members in a dungeon.")
if err != nil {
t.Fatalf("build err: %v", err)
}
@@ -192,10 +192,31 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
for _, needle := range []string{
"Transcript description (background context only):",
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(combined, needle) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
})
}
}
func TestPromptBuildersOmitTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters"}}
msgs, err := BuildSpokenFormPlausibilityMessages(payload, " ")
if err != nil {
t.Fatalf("build err: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("did not expect empty transcript description section in prompt")
}
}
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},