eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Move LLM validators to PromptKit prompt variables

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 02:07:17 +00:00
parent f577847ca2
commit a4a72168d5
17 changed files with 310 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
internal/framework/validators/llm_validators_test.go
View File

@@ -15,6 +15,8 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/internal/prompts"
)
type fakeStructuredLLMClient struct {
@@ -109,8 +111,8 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req St
}
func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
cfg := config.Default()
cfg.ValidationMaxPromptTokens = 10000
cfg := config.Config{}
cfg.Validation.MaxPromptTokens = 10000
return Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple.", Categories: []string{"narration"}}}},
CandidateProposal: candidates,
@@ -163,66 +165,12 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
}
}
func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters", Categories: []string{"narration"}}}
tests := []struct {
name string
build func([]LLMValidationItem, string) ([]LLMMessage, error)
mustHas []string
}{
{"spoken_form", BuildSpokenFormPlausibilityMessages, []string{"plausible spoken-form", "correction_index", "original_segment_text", "corrected_segment_text"}},
{"meaning_reversal", BuildMeaningReversalMessages, []string{"meaning reversals", "correction_index", "original_segment_text", "corrected_segment_text"}},
{"editorial", BuildEditorialMessages, []string{"acceptable editorial revision", "correction_index", "original_segment_text", "corrected_segment_text"}},
{"grammar_review", BuildGrammarReviewMessages, []string{"acceptable editorial revision", "correction_index"}},
{"spoken_word", BuildSpokenWordReviewMessages, []string{"acceptable editorial revision", "correction_index"}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
msgs, err := tt.build(payload, "Discussion among party members in a dungeon.")
if err != nil {
t.Fatalf("build err: %v", err)
}
if len(msgs) != 2 {
t.Fatalf("expected 2 messages, got %d", len(msgs))
}
combined := msgs[0].Content + "\n" + msgs[1].Content
for _, needle := range tt.mustHas {
if !strings.Contains(combined, needle) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
for _, needle := range []string{
"Transcript description (background context only):",
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(combined, needle) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
})
}
}
func TestPromptBuildersOmitTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters"}}
msgs, err := BuildSpokenFormPlausibilityMessages(payload, " ")
if err != nil {
t.Fatalf("build err: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("did not expect empty transcript description section in prompt")
}
}
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"},
}}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("new validator error: %v", err)
}
@@ -238,19 +186,15 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
if len(client.calls) != 1 {
t.Fatalf("expected one LLM call, got %d", len(client.calls))
}
if client.calls[0].ResponseSchema == nil {
t.Fatalf("expected response schema on structured validation request")
}
want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
gotSchema := client.calls[0].ResponseSchema
if gotSchema.ID != want.ID || gotSchema.Version != want.Version || gotSchema.Name != want.Name || gotSchema.SHA256 != want.SHA256 {
t.Fatalf("unexpected validator response schema metadata: got=%+v want=%+v", *gotSchema, want)
call := client.calls[0]
if call.Schema != responseschema.ValidatorDecisionSetKey || call.PromptID != prompts.PromptIDValidatorSpokenFormPlausibility || call.Context.Role != "validation" || call.Context.ValidatorName != v.Name() || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") {
t.Fatalf("unexpected validator call: %+v", call)
}
}
func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
@@ -265,26 +209,20 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
}
}
func TestLLMBackedValidatorProviderMalformedEnvelopeRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{err: errors.New("provider response assistant message content is empty")}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
if err != nil {
t.Fatalf("expected malformed provider envelope downgrade, got %v", err)
}
if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", res.Warnings)
result, err := v.Validate(context.Background(), req)
if err == nil || len(result.Warnings) != 0 {
t.Fatalf("protocol error was downgraded: %+v, %v", result, err)
}
}
func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
@@ -301,7 +239,7 @@ func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
@@ -315,7 +253,7 @@ func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
res, err := v.Validate(context.Background(), req)
@@ -331,7 +269,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
huge := strings.Repeat("gestures ", 200)
req := Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{
@@ -353,8 +291,8 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
}
req.LLMClient = client
cfg := config.Default()
cfg.ValidationMaxPromptTokens = 200
cfg := config.Config{}
cfg.Validation.MaxPromptTokens = 200
req.Config = &cfg
res, err := v.Validate(context.Background(), req)
@@ -381,7 +319,7 @@ func TestLLMBackedValidatorRespectsSchedulerConcurrency(t *testing.T) {
entered: make(chan struct{}, 16),
release: make(chan struct{}),
}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("new validator error: %v", err)
}
@@ -416,7 +354,7 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model")
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("new validator error: %v", err)
}
@@ -439,19 +377,7 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
}
promptMap, ok := metadata["prompt_metadata"].(map[string]any)
if !ok {
t.Fatalf("expected prompt_metadata map, got %T", metadata["prompt_metadata"])
}
if promptMap["prompt_id"] != "validators.spoken_form_plausibility" || promptMap["prompt_version"] != "v1" || promptMap["prompt_source"] != "builtin" {
t.Fatalf("unexpected prompt metadata: %v", promptMap)
}
if _, ok := promptMap["embedded_path"].(string); !ok {
t.Fatalf("expected embedded_path in prompt metadata: %v", promptMap)
}
if _, ok := promptMap["sha256"].(string); !ok {
t.Fatalf("expected sha256 in prompt metadata: %v", promptMap)
}
}
func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) {
@@ -470,3 +396,38 @@ func waitForValidationEntries(t *testing.T, entered <-chan struct{}, want int) {
t.Fatalf("timed out waiting for %d validator calls to enter (got %d)", want, got)
}
}
func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
cases := []struct {
kind LLMValidatorType
prompt string
}{
{LLMValidatorTypeSpokenFormPlausibility, prompts.PromptIDValidatorSpokenFormPlausibility},
{LLMValidatorTypeMeaningReversal, prompts.PromptIDValidatorMeaningReversalReview},
{LLMValidatorTypeEditorialReview, prompts.PromptIDValidatorEditorialReview},
{LLMValidatorTypeGrammarReview, prompts.PromptIDValidatorGrammarReview},
{LLMValidatorTypeSpokenWordReview, prompts.PromptIDValidatorSpokenWordReview},
}
for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil {
t.Fatal(err)
}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
req.Config.Context.Description = " Fantasy scene "
if _, err := validator.Validate(context.Background(), req); err != nil {
t.Fatal(err)
}
if len(client.calls) != 1 {
t.Fatalf("calls=%d", len(client.calls))
}
call := client.calls[0]
if call.PromptID != tc.prompt || call.PromptVersion != prompts.VersionV1 || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") {
t.Fatalf("mapping: %+v", call)
}
})
}
}