eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Centralize validator classification and malformed output handling

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-23 18:02:53 +00:00
parent 84be774b34
commit 99391cd18b
12 changed files with 255 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
internal/cli/review_artifacts.go
View File

@@ -5,6 +5,7 @@ import (
"sort"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata"
)
const (
@@ -48,12 +49,6 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
}
entries := make([]correctionLedgerEntry, 0)
llmBacked := map[string]bool{
"spoken_form_plausibility": true,
"meaning_reversal_review": true,
"editorial_review": true,
}
for _, module := range runOutput.ModuleResults {
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
for _, decision := range module.ValidatorDecisions {
@@ -72,8 +67,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
AppliedCorrectedText: change.CorrectedText,
ReplacementPolicy: string(module.ReplacementPolicy),
Disposition: correctionDispositionApplied,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
})
}
for _, change := range module.SkippedChanges {
@@ -89,8 +84,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
Disposition: correctionDispositionSkipped,
DispositionReasonCode: string(change.SkipReason),
DispositionMessage: change.Message,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true),
})
}
for _, rejection := range module.ValidatorRejected {
@@ -106,8 +101,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
Disposition: correctionDispositionRejected,
DispositionReasonCode: rejection.ReasonCode,
DispositionMessage: rejection.Message,
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false, llmBacked),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true, llmBacked),
DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false),
LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true),
})
}
if module.Status == runner.ModuleStatusFailed {
@@ -135,13 +130,14 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor
return entries
}
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool, llmBacked map[string]bool) []ledgerValidatorDecisionRecord {
func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord {
if len(in) == 0 {
return nil
}
out := make([]ledgerValidatorDecisionRecord, 0, len(in))
for _, decision := range in {
if llmBacked[decision.ValidatorName] != wantLLM {
isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked
if isLLMBacked != wantLLM {
continue
}
out = append(out, ledgerValidatorDecisionRecord{

46
internal/cli/review_artifacts_test.go Normal file
View File

@@ -0,0 +1,46 @@
package cli
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
)
func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) {
output := &runner.RunOutput{
ModuleResults: []runner.ModuleResult{
{
ModuleKey: "glossary",
ModuleInstance: "glossary",
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ValidatorDecisions: []runner.ValidatorDecisionRecord{
{ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
{ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"},
},
AppliedChanges: []proposals.AppliedChange{
{
ProposalIndex: 3,
ModuleKey: "glossary",
ModuleInstance: "glossary",
TargetSegmentID: 1,
OriginalText: "gestures",
CorrectedText: "Jesters",
},
},
},
},
}
ledger := buildCorrectionLedger("/tmp/audita-run-id", output)
if len(ledger) != 1 {
t.Fatalf("expected one ledger entry, got %d", len(ledger))
}
entry := ledger[0]
if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" {
t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults)
}
if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" {
t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults)
}
}