eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Track unavailable validator decisions in reports and debug records

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 11:28:44 +00:00
parent d726ea8f6d
commit 22f0932389
11 changed files with 169 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
internal/framework/processreport/correction_ledger.go
View File

@@ -6,12 +6,14 @@ import (
"sort"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
const (
correctionDispositionApplied = "applied"
correctionDispositionRejected = "rejected"
correctionDispositionUnavailable = "unavailable"
correctionDispositionSkipped = "skipped"
correctionDispositionFailed = "failed"
correctionDispositionInterrupted = "interrupted"
@@ -40,10 +42,11 @@ type CorrectionLedgerEntry struct {
}
type LedgerValidatorDecisionRecord struct {
ValidatorKey string `json:"validator_key"`
Approved bool `json:"approved"`
ReasonCode string `json:"reason_code"`
Message string `json:"message,omitempty"`
ValidatorKey string `json:"validator_key"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
ReasonCode string `json:"reason_code"`
Message string `json:"message,omitempty"`
}
func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry {
@@ -101,6 +104,10 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
for _, rejection := range module.ValidatorRejected {
disposition := correctionDispositionRejected
if rejection.Disposition == validators.DispositionUnavailable {
disposition = correctionDispositionUnavailable
}
seen[identity(module.ModuleInstance, rejection.ProposalIndex)] = true
entries = append(entries, CorrectionLedgerEntry{
RunID: runID,
@@ -111,7 +118,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
OriginalText: rejection.OriginalText,
ProposedCorrectedText: &rejection.CorrectedText,
ReplacementPolicy: string(module.ReplacementPolicy),
Disposition: correctionDispositionRejected,
Disposition: disposition,
DispositionReasonCode: rejection.ReasonCode,
DispositionMessage: rejection.Message,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false),
@@ -184,6 +191,7 @@ func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []
out = append(out, LedgerValidatorDecisionRecord{
ValidatorKey: decision.ValidatorName,
Approved: decision.Approved,
Disposition: decision.Disposition,
ReasonCode: decision.ReasonCode,
Message: decision.Message,
})

34
internal/framework/processreport/correction_ledger_test.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
@@ -16,8 +17,8 @@ func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(
ModuleInstance: "glossary",
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ValidatorDecisions: []runner.ValidatorDecisionRecord{
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, Disposition: validators.DispositionApproved, ReasonCode: "approved"},
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, Disposition: validators.DispositionApproved, ReasonCode: "approved"},
},
AppliedChanges: []proposals.AppliedChange{
{
@@ -44,10 +45,10 @@ func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(
if entry.RunID != "audita-run-id" || entry.Disposition != "applied" || entry.AppliedCorrectedText == nil || *entry.AppliedCorrectedText != "Jesters" {
t.Fatalf("unexpected applied ledger entry: %+v", entry)
}
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" || entry.DeterministicValidatorResults[0].Disposition != validators.DispositionApproved {
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
}
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" || entry.LLMValidatorResults[0].Disposition != validators.DispositionApproved {
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
}
}
@@ -76,6 +77,7 @@ func TestBuildCorrectionLedgerPreservesDispositionPolicy(t *testing.T) {
TargetSegmentID: 8,
OriginalText: "before",
CorrectedText: "after",
Disposition: validators.DispositionRejected,
ReasonCode: "protected_term",
Message: "blocked",
},
@@ -117,7 +119,7 @@ func TestCorrectionLedgerIncludesUnappliedCandidateAfterFatalValidation(t *testi
output := &runner.RunOutput{
Candidates: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"}, ProposalMetadata: proposals.ProposalMetadata{ModuleKey: "grammar", ModuleInstance: "grammar", ProposalIndex: 2}}},
ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar", Status: runner.ModuleStatusFailed,
ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "spoken_form_plausibility", ProposalIndex: 2, Approved: false, ReasonCode: "transport_failed"}}}},
ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "spoken_form_plausibility", ProposalIndex: 2, Approved: false, Disposition: validators.DispositionUnavailable, ReasonCode: "transport_failed"}}}},
}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
if len(ledger) != 2 {
@@ -127,7 +129,7 @@ func TestCorrectionLedgerIncludesUnappliedCandidateAfterFatalValidation(t *testi
if entry.ProposalIndex != 2 {
continue
}
if entry.Disposition != correctionDispositionInterrupted || entry.OriginalText != "old" || len(entry.LLMValidatorResults) != 1 {
if entry.Disposition != correctionDispositionInterrupted || entry.OriginalText != "old" || len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].Disposition != validators.DispositionUnavailable {
t.Fatalf("candidate evidence lost: %+v", entry)
}
return
@@ -140,9 +142,16 @@ func TestCorrectionLedgerSerializesEmptyReplacementByDisposition(t *testing.T) {
ModuleResults: []runner.ModuleResult{
{
ModuleKey: "spoken_word", ModuleInstance: "spoken_word", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 0, TargetSegmentID: 1, OriginalText: "uh ", CorrectedText: ""}},
ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 1, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: "", ReasonCode: "llm_rejected"}},
SkippedChanges: []proposals.SkippedChange{{ProposalIndex: 2, TargetSegmentID: 3, OriginalText: "uh ", CorrectedText: "", SkipReason: proposals.SkipReasonEmptyResultingText}},
AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 0, TargetSegmentID: 1, OriginalText: "uh ", CorrectedText: ""}},
ValidatorRejected: []runner.ValidatorRejectedChange{
{ProposalIndex: 1, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: "", Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected"},
{ProposalIndex: 4, TargetSegmentID: 5, OriginalText: "uh ", CorrectedText: "", Disposition: validators.DispositionUnavailable, ReasonCode: validators.ReasonMissingValidationDecision},
},
ValidatorDecisions: []runner.ValidatorDecisionRecord{
{ValidatorName: "editorial_review", ProposalIndex: 1, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected"},
{ValidatorName: "editorial_review", ProposalIndex: 4, Disposition: validators.DispositionUnavailable, ReasonCode: validators.ReasonMissingValidationDecision},
},
SkippedChanges: []proposals.SkippedChange{{ProposalIndex: 2, TargetSegmentID: 3, OriginalText: "uh ", CorrectedText: "", SkipReason: proposals.SkipReasonEmptyResultingText}},
},
{ModuleKey: "grammar", ModuleInstance: "grammar", Status: runner.ModuleStatusFailed, ErrorMessage: "failed"},
},
@@ -167,6 +176,7 @@ func TestCorrectionLedgerSerializesEmptyReplacementByDisposition(t *testing.T) {
}{
{correctionDispositionApplied, true, true},
{correctionDispositionRejected, true, false},
{correctionDispositionUnavailable, true, false},
{correctionDispositionSkipped, true, false},
{correctionDispositionInterrupted, true, false},
{correctionDispositionFailed, false, false},
@@ -180,5 +190,11 @@ func TestCorrectionLedgerSerializesEmptyReplacementByDisposition(t *testing.T) {
if proposedPresent != tc.proposed || appliedPresent != tc.applied || (proposedPresent && string(proposed) != `""`) || (appliedPresent && string(applied) != `""`) {
t.Fatalf("incorrect serialized replacement fields for %s: %+v", tc.disposition, fields)
}
if tc.disposition == correctionDispositionUnavailable || tc.disposition == correctionDispositionRejected {
var decisions []LedgerValidatorDecisionRecord
if err := json.Unmarshal(fields["llm_validator_decisions"], &decisions); err != nil || len(decisions) != 1 || string(decisions[0].Disposition) != tc.disposition {
t.Fatalf("incorrect validator disposition for %s: %+v, %v", tc.disposition, decisions, err)
}
}
}
}

12
internal/framework/processreport/partial_validation_test.go
View File

@@ -51,6 +51,10 @@ func (c *interruptedValidationClient) CompleteStructured(_ context.Context, req
return structuredoutput.Response{}, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "validation interrupted", Cause: c.cause}
}
index := *req.Context.BatchIndex
if index == 0 {
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{}
return structuredoutput.Response{}, nil
}
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Confidence: 1, Reason: "completed assessment"}}}
return structuredoutput.Response{}, nil
}
@@ -89,11 +93,15 @@ func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) {
}
for index := 0; index < 2; index++ {
entry := candidates[index]
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].Message != "completed assessment" || entry.LLMValidatorResults[0].Approved != (index == 1) {
wantDisposition := validators.DispositionUnavailable
if index == 1 {
wantDisposition = validators.DispositionApproved
}
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].Approved != (index == 1) || entry.LLMValidatorResults[0].Disposition != wantDisposition {
t.Errorf("lost batch %d decision: %+v", index, entry)
}
}
if candidates[0].Disposition != correctionDispositionRejected || candidates[1].Disposition != correctionDispositionInterrupted || candidates[2].Disposition != correctionDispositionInterrupted {
if candidates[0].Disposition != correctionDispositionUnavailable || candidates[1].Disposition != correctionDispositionInterrupted || candidates[2].Disposition != correctionDispositionInterrupted {
t.Fatalf("wrong partial dispositions: %+v", ledger)
}
}

4
internal/framework/processreport/report.go
View File

@@ -151,7 +151,7 @@ func mapRecovery(unit runner.WorkUnitOutcome, operation outcomes.Operation) repo
return reporting.RecoveryRecord{StageIndex: unit.ModuleIndex, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
Operation: string(operation.Reference.Kind), Validator: operation.Reference.ValidatorName, BatchIndex: operation.Reference.BatchIndex,
Reasons: reasons, Message: "Model operation succeeded after recovery or proposal filtering.",
Reasons: reasons, Message: "Model operation succeeded after recovery or response filtering.",
ExecutionCounts: mapCounts(operation.Counts), FilteredProposals: filtered}
}
@@ -195,7 +195,7 @@ func safeMessage(reason outcomes.ReasonCode) string {
case outcomes.ReasonValidationInputTooLarge:
return "Validation input exceeded its size limit."
case outcomes.ReasonValidationDecisionsInvalid:
return "Validation decisions were unusable."
return "Some required validation decisions were unavailable."
case outcomes.ReasonCanceled:
return "Processing was canceled."
case outcomes.ReasonSetupFailure:

55
internal/framework/processreport/report_test.go
View File

@@ -57,6 +57,61 @@ func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) {
}
}
func TestBuildReportsUnavailableDecisionsAndFilteredExtras(t *testing.T) {
started := time.Date(2026, 9, 25, 12, 0, 0, 0, time.UTC)
completed := started.Add(time.Second)
failedBatch, cleanBatch := 0, 1
output := &runner.RunOutput{InventoryAvailable: true, WorkUnits: []runner.WorkUnitOutcome{{
ModuleKey: "grammar", ModuleInstance: "grammar", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 1},
State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "editorial_review", BatchIndex: &failedBatch}, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, FailureReason: outcomes.ReasonValidationDecisionsInvalid, ProposalIndices: []int{3, 1}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "editorial_review", BatchIndex: &cleanBatch}, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}, Succeeded: true, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryValidationDecisionsFiltered}},
},
}}}
snapshot := Build(BuildInput{RunID: "partial", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, RunOutput: output})
if snapshot.Summary.WorkUnitsWithErrors == nil || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 0 || *snapshot.Summary.LLMOperations != 3 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || len(snapshot.Errors[0].Failures) != 1 || !reflect.DeepEqual(snapshot.Errors[0].Failures[0].ProposalIndices, []int{1, 3}) || snapshot.Errors[0].Failures[0].Reason != string(outcomes.ReasonValidationDecisionsInvalid) || snapshot.Errors[0].Failures[0].ExecutionCounts.StructuralRepairs == nil || *snapshot.Errors[0].Failures[0].ExecutionCounts.StructuralRepairs != 1 {
t.Fatalf("failure attribution: %+v", snapshot.Errors)
}
if len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{string(outcomes.RecoveryValidationDecisionsFiltered)}) || len(snapshot.Recoveries[0].FilteredProposals) != 0 {
t.Fatalf("recovery attribution: %+v", snapshot.Recoveries)
}
for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} {
encoded, err := reporting.YAML(document)
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := yaml.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if decoded["version"] != 1 || decoded["status"] != "completed" {
t.Fatalf("YAML envelope: %#v", decoded)
}
summary := decoded["summary"].(map[string]any)
if summary["work_units_with_errors"] != 1 || summary["recoveries"] != 1 || summary["proposal_responses_with_filtering"] != 0 {
t.Fatalf("YAML summary: %#v", summary)
}
if errors, ok := decoded["errors"].([]any); ok {
failure := errors[0].(map[string]any)["failures"].([]any)[0].(map[string]any)
if !reflect.DeepEqual(failure["proposal_indices"], []any{1, 3}) {
t.Fatalf("YAML failure indices: %#v", failure)
}
}
if recoveries, ok := decoded["recoveries"].([]any); ok {
recovery := recoveries[0].(map[string]any)
filtered, hasFiltered := recovery["filtered_proposals"]
if !reflect.DeepEqual(recovery["reasons"], []any{string(outcomes.RecoveryValidationDecisionsFiltered)}) || hasFiltered && len(filtered.([]any)) != 0 {
t.Fatalf("YAML recovery: %#v", recovery)
}
}
}
}
func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
start := time.Now().UTC()
initial := Build(BuildInput{RunID: "r", Status: reporting.StatusInProgress, StartedAt: start})