Add utilization diagnostics and correction ledger

2026-05-13 19:03:37 +00:00
parent 037121e9ce
commit ff4ed82239
15 changed files with 969 additions and 13 deletions
--- a/internal/cli/review_artifacts.go
+++ b/internal/cli/review_artifacts.go
@@ -0,0 +1,157 @@
+package cli
+
+import (
+	"path/filepath"
+	"sort"
+
+	"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
+)
+
+const (
+	correctionDispositionApplied  = "applied"
+	correctionDispositionRejected = "rejected"
+	correctionDispositionSkipped  = "skipped"
+	correctionDispositionFailed   = "failed"
+)
+
+type correctionLedgerEntry struct {
+	RunID                         string                          `json:"run_id,omitempty"`
+	ModuleKey                     string                          `json:"module_key"`
+	ModuleInstance                string                          `json:"module_instance"`
+	ProposalIndex                 int                             `json:"proposal_index"`
+	SegmentID                     int                             `json:"segment_id,omitempty"`
+	OriginalText                  string                          `json:"original_text,omitempty"`
+	ProposedCorrectedText         string                          `json:"proposed_corrected_text,omitempty"`
+	AppliedCorrectedText          string                          `json:"applied_corrected_text,omitempty"`
+	ReplacementPolicy             string                          `json:"replacement_policy,omitempty"`
+	Disposition                   string                          `json:"disposition"`
+	DispositionReasonCode         string                          `json:"disposition_reason_code,omitempty"`
+	DispositionMessage            string                          `json:"disposition_message,omitempty"`
+	DeterministicValidatorResults []ledgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
+	LLMValidatorResults           []ledgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
+}
+
+type ledgerValidatorDecisionRecord struct {
+	ValidatorKey string `json:"validator_key"`
+	Approved     bool   `json:"approved"`
+	ReasonCode   string `json:"reason_code"`
+	Message      string `json:"message,omitempty"`
+}
+
+func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []correctionLedgerEntry {
+	if runOutput == nil || len(runOutput.ModuleResults) == 0 {
+		return nil
+	}
+	runID := ""
+	if runDirPath != "" {
+		runID = filepath.Base(runDirPath)
+	}
+
+	entries := make([]correctionLedgerEntry, 0)
+	llmBacked := map[string]bool{
+		"spoken_form_plausibility": true,
+		"meaning_reversal_review":  true,
+		"editorial_review":         true,
+		"grammar_review":           true,
+		"spoken_word_review":       true,
+	}
+
+	for _, module := range runOutput.ModuleResults {
+		decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
+		for _, decision := range module.ValidatorDecisions {
+			decisionsByProposal[decision.ProposalIndex] = append(decisionsByProposal[decision.ProposalIndex], decision)
+		}
+
+		for _, change := range module.AppliedChanges {
+			entries = append(entries, correctionLedgerEntry{
+				RunID:                         runID,
+				ModuleKey:                     module.ModuleKey,
+				ModuleInstance:                module.ModuleInstance,
+				ProposalIndex:                 change.ProposalIndex,
+				SegmentID:                     change.TargetSegmentID,
+				OriginalText:                  change.OriginalText,
+				ProposedCorrectedText:         change.CorrectedText,
+				AppliedCorrectedText:          change.CorrectedText,
+				ReplacementPolicy:             string(module.ReplacementPolicy),
+				Disposition:                   correctionDispositionApplied,
+				DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
+				LLMValidatorResults:           filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
+			})
+		}
+		for _, change := range module.SkippedChanges {
+			entries = append(entries, correctionLedgerEntry{
+				RunID:                         runID,
+				ModuleKey:                     module.ModuleKey,
+				ModuleInstance:                module.ModuleInstance,
+				ProposalIndex:                 change.ProposalIndex,
+				SegmentID:                     change.TargetSegmentID,
+				OriginalText:                  change.OriginalText,
+				ProposedCorrectedText:         change.CorrectedText,
+				ReplacementPolicy:             string(module.ReplacementPolicy),
+				Disposition:                   correctionDispositionSkipped,
+				DispositionReasonCode:         string(change.SkipReason),
+				DispositionMessage:            change.Message,
+				DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked),
+				LLMValidatorResults:           filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked),
+			})
+		}
+		for _, rejection := range module.ValidatorRejected {
+			entries = append(entries, correctionLedgerEntry{
+				RunID:                         runID,
+				ModuleKey:                     module.ModuleKey,
+				ModuleInstance:                module.ModuleInstance,
+				ProposalIndex:                 rejection.ProposalIndex,
+				SegmentID:                     rejection.TargetSegmentID,
+				OriginalText:                  rejection.OriginalText,
+				ProposedCorrectedText:         rejection.CorrectedText,
+				ReplacementPolicy:             string(module.ReplacementPolicy),
+				Disposition:                   correctionDispositionRejected,
+				DispositionReasonCode:         rejection.ReasonCode,
+				DispositionMessage:            rejection.Message,
+				DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false, llmBacked),
+				LLMValidatorResults:           filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true, llmBacked),
+			})
+		}
+		if module.Status == runner.ModuleStatusFailed {
+			entries = append(entries, correctionLedgerEntry{
+				RunID:                 runID,
+				ModuleKey:             module.ModuleKey,
+				ModuleInstance:        module.ModuleInstance,
+				Disposition:           correctionDispositionFailed,
+				DispositionReasonCode: "module_failed",
+				DispositionMessage:    module.ErrorMessage,
+			})
+		}
+	}
+
+	sort.SliceStable(entries, func(i, j int) bool {
+		if entries[i].ModuleInstance != entries[j].ModuleInstance {
+			return entries[i].ModuleInstance < entries[j].ModuleInstance
+		}
+		if entries[i].ProposalIndex != entries[j].ProposalIndex {
+			return entries[i].ProposalIndex < entries[j].ProposalIndex
+		}
+		return entries[i].Disposition < entries[j].Disposition
+	})
+
+	return entries
+}
+
+func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool, llmBacked map[string]bool) []ledgerValidatorDecisionRecord {
+	if len(in) == 0 {
+		return nil
+	}
+	out := make([]ledgerValidatorDecisionRecord, 0, len(in))
+	for _, decision := range in {
+		if llmBacked[decision.ValidatorName] != wantLLM {
+			continue
+		}
+		out = append(out, ledgerValidatorDecisionRecord{
+			ValidatorKey: decision.ValidatorName,
+			Approved:     decision.Approved,
+			ReasonCode:   decision.ReasonCode,
+			Message:      decision.Message,
+		})
+	}
+	return out
+}