eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Fix the five resilience review findings

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 21:48:18 +00:00
parent 9fc4de22b7
commit ec78b5a5db
19 changed files with 550 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/framework/validators/llm_validators.go
View File

@@ -151,7 +151,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
artifacts := InteractionArtifacts{}
if req.DiagnosticsWriter != nil {
artifacts, _ = req.DiagnosticsWriter.WriteInteraction(
var diagnosticErr error
artifacts, diagnosticErr = req.DiagnosticsWriter.WriteInteraction(
stage,
map[string]any{
"validator_name": v.name,
@@ -164,6 +165,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
errPayload(err),
)
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr))
}
}
if err != nil {
var failure *structuredoutput.Failure

17
internal/framework/validators/llm_validators_test.go
View File

@@ -23,6 +23,7 @@ type fakeStructuredLLMClient struct {
}
type captureValidationDiagnosticsWriter struct {
err error
lastRequestMetadata any
}
@@ -32,7 +33,7 @@ func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requ
_ = responsePayload
_ = errorPayload
w.lastRequestMetadata = requestMetadata
return InteractionArtifacts{}, nil
return InteractionArtifacts{}, w.err
}
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
@@ -321,3 +322,17 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
})
}
}
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
result, err := validator.Validate(context.Background(), req)
if err != nil || !errors.Is(reported, cause) || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
}
}

39
internal/framework/validators/models.go
View File

@@ -29,17 +29,19 @@ const (
// Request is the runtime input shared by deterministic validators.
type Request struct {
SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ValidatorOrder *int `json:"validator_order,omitempty"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Glossary *schema.Glossary `json:"-"`
Config *config.Config `json:"-"`
LLMClient StructuredLLMClient `json:"-"`
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ValidatorOrder *int `json:"validator_order,omitempty"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Glossary *schema.Glossary `json:"-"`
Config *config.Config `json:"-"`
LLMClient StructuredLLMClient `json:"-"`
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
}
// Decision is one validator decision for one proposal index.
@@ -62,7 +64,8 @@ const (
DispositionUnavailable DecisionDisposition = "unavailable"
)
// Result is one validator output containing exactly one decision per proposal index.
// Result contains one decision per proposal index on success. Fatal returns may
// include completed partial decisions as evidence, never as application approval.
type Result struct {
ValidatorName string `json:"validator_name"`
Decisions []Decision `json:"decisions"`
@@ -88,6 +91,16 @@ type Validator interface {
// EnforceDecisionCardinality verifies every candidate proposal index receives exactly one decision.
func EnforceDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal, decisions []Decision) error {
return enforceDecisionCardinality(candidate, decisions, true)
}
// EnforcePartialDecisionCardinality accepts missing decisions on fatal returns,
// but still rejects duplicate and unknown proposal identities.
func EnforcePartialDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal, decisions []Decision) error {
return enforceDecisionCardinality(candidate, decisions, false)
}
func enforceDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal, decisions []Decision, requireAll bool) error {
expected := make(map[int]struct{}, len(candidate))
for _, c := range candidate {
expected[c.ProposalIndex] = struct{}{}
@@ -105,7 +118,7 @@ func EnforceDecisionCardinality(candidate []proposals.EnrichedCorrectionProposal
}
for idx := range expected {
if seen[idx] == 0 {
if requireAll && seen[idx] == 0 {
return fmt.Errorf("missing decision proposal index %d", idx)
}
}