eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Track unavailable validator decisions in reports and debug records

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 11:28:44 +00:00
parent d726ea8f6d
commit 22f0932389
11 changed files with 169 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/framework/validators/llm_validators_test.go
View File

@@ -25,6 +25,7 @@ type fakeStructuredLLMClient struct {
type captureValidationDiagnosticsWriter struct {
err error
path string
lastRequestMetadata any
lastResponsePayload any
}
@@ -35,7 +36,7 @@ func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requ
_ = errorPayload
w.lastRequestMetadata = requestMetadata
w.lastResponsePayload = responsePayload
return InteractionArtifacts{}, w.err
return InteractionArtifacts{ResponsePayloadPath: w.path}, w.err
}
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
@@ -233,7 +234,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
writer := &captureValidationDiagnosticsWriter{}
writer := &captureValidationDiagnosticsWriter{path: "validation-response.json"}
v, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "actors"), mk(18, "gestures", "figures")})
req.LLMClient = client
@@ -244,7 +245,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
}
for i, want := range tc.want {
got := result.Decisions[i]
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) {
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) {
t.Fatalf("decision[%d]=%+v want=%+v", i, got, want)
}
if got.ReasonCode == "llm_rejected" && got.Message != "editorial concern" {
@@ -262,6 +263,10 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
t.Fatalf("recovery reason=%+v", op)
}
payload := writer.lastResponsePayload.(map[string]any)
decoded := payload["decoded"].(LLMValidationResponse)
if len(decoded.Validations) != len(tc.entries) {
t.Fatalf("supplied model decisions lost from diagnostic: %+v", decoded)
}
reconciliation := payload["reconciliation"].(validationReconciliation)
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
t.Fatalf("diagnostic reconciliation=%+v", reconciliation)