eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Track unavailable validator decisions in reports and debug records

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 11:28:44 +00:00
parent d726ea8f6d
commit 22f0932389
11 changed files with 169 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
internal/framework/processreport/report_test.go
View File

@@ -57,6 +57,61 @@ func TestBuildPartialUnitsAndCombinedRecovery(t *testing.T) {
}
}
func TestBuildReportsUnavailableDecisionsAndFilteredExtras(t *testing.T) {
started := time.Date(2026, 9, 25, 12, 0, 0, 0, time.UTC)
completed := started.Add(time.Second)
failedBatch, cleanBatch := 0, 1
output := &runner.RunOutput{InventoryAvailable: true, WorkUnits: []runner.WorkUnitOutcome{{
ModuleKey: "grammar", ModuleInstance: "grammar", Section: contracts.SectionMetadata{Index: 0, StartSegmentID: 1, EndSegmentID: 1},
State: outcomes.WorkCompleted, CorrectionsApplied: 1, Operations: []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "editorial_review", BatchIndex: &failedBatch}, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}, FailureReason: outcomes.ReasonValidationDecisionsInvalid, ProposalIndices: []int{3, 1}},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation, ValidatorName: "editorial_review", BatchIndex: &cleanBatch}, Counts: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}, Succeeded: true, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryValidationDecisionsFiltered}},
},
}}}
snapshot := Build(BuildInput{RunID: "partial", Status: reporting.StatusCompleted, StartedAt: started, CompletedAt: &completed,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, RunOutput: output})
if snapshot.Summary.WorkUnitsWithErrors == nil || *snapshot.Summary.WorkUnitsWithErrors != 1 || *snapshot.Summary.CorrectionsApplied != 1 || *snapshot.Summary.Recoveries != 1 || *snapshot.Summary.ProposalResponsesWithFiltering != 0 || *snapshot.Summary.LLMOperations != 3 {
t.Fatalf("summary: %+v", snapshot.Summary)
}
if len(snapshot.Errors) != 1 || len(snapshot.Errors[0].Failures) != 1 || !reflect.DeepEqual(snapshot.Errors[0].Failures[0].ProposalIndices, []int{1, 3}) || snapshot.Errors[0].Failures[0].Reason != string(outcomes.ReasonValidationDecisionsInvalid) || snapshot.Errors[0].Failures[0].ExecutionCounts.StructuralRepairs == nil || *snapshot.Errors[0].Failures[0].ExecutionCounts.StructuralRepairs != 1 {
t.Fatalf("failure attribution: %+v", snapshot.Errors)
}
if len(snapshot.Recoveries) != 1 || !reflect.DeepEqual(snapshot.Recoveries[0].Reasons, []string{string(outcomes.RecoveryValidationDecisionsFiltered)}) || len(snapshot.Recoveries[0].FilteredProposals) != 0 {
t.Fatalf("recovery attribution: %+v", snapshot.Recoveries)
}
for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} {
encoded, err := reporting.YAML(document)
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := yaml.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
if decoded["version"] != 1 || decoded["status"] != "completed" {
t.Fatalf("YAML envelope: %#v", decoded)
}
summary := decoded["summary"].(map[string]any)
if summary["work_units_with_errors"] != 1 || summary["recoveries"] != 1 || summary["proposal_responses_with_filtering"] != 0 {
t.Fatalf("YAML summary: %#v", summary)
}
if errors, ok := decoded["errors"].([]any); ok {
failure := errors[0].(map[string]any)["failures"].([]any)[0].(map[string]any)
if !reflect.DeepEqual(failure["proposal_indices"], []any{1, 3}) {
t.Fatalf("YAML failure indices: %#v", failure)
}
}
if recoveries, ok := decoded["recoveries"].([]any); ok {
recovery := recoveries[0].(map[string]any)
filtered, hasFiltered := recovery["filtered_proposals"]
if !reflect.DeepEqual(recovery["reasons"], []any{string(outcomes.RecoveryValidationDecisionsFiltered)}) || hasFiltered && len(filtered.([]any)) != 0 {
t.Fatalf("YAML recovery: %#v", recovery)
}
}
}
}
func TestBuildInitializationNoWorkAndRunFailure(t *testing.T) {
start := time.Now().UTC()
initial := Build(BuildInput{RunID: "r", Status: reporting.StatusInProgress, StartedAt: start})