eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Simplify references and output instructions in LLM prompts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 12:38:23 +00:00
parent 325c4ea7d3
commit 011b021d47
31 changed files with 248 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/framework/validators/llm_validators_test.go
View File

@@ -152,8 +152,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
}}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
@@ -224,13 +224,14 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
unavailable []int
ignored []int
}{
{"missing and extra", []LLMValidationDecision{approve(7), reject(18), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"conflicting duplicate", []LLMValidationDecision{approve(7), approve(11), reject(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(7), approve(11), approve(11), reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(18), approve(7), approve(99), approve(11)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(18), approve(11), approve(7)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(7), {CorrectionIndex: 11, Approved: true, Confidence: 1.2}, reject(18)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
} {
t.Run(tc.name, func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
@@ -337,7 +338,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
@@ -376,7 +377,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}}
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil {
t.Fatal(err)
@@ -401,7 +402,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }