eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Make debug evidence opt-in and retain all candidates

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:41:10 +00:00
parent a80eed4745
commit 87b8d9ad5c
10 changed files with 246 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
internal/framework/processreport/correction_ledger.go
View File

@@ -1,6 +1,7 @@
package processreport
import (
"fmt"
"path/filepath"
"sort"
@@ -9,10 +10,11 @@ import (
)
const (
correctionDispositionApplied = "applied"
correctionDispositionRejected = "rejected"
correctionDispositionSkipped = "skipped"
correctionDispositionFailed = "failed"
correctionDispositionApplied = "applied"
correctionDispositionRejected = "rejected"
correctionDispositionSkipped = "skipped"
correctionDispositionFailed = "failed"
correctionDispositionInterrupted = "interrupted"
)
type CorrectionLedgerInput struct {
@@ -55,6 +57,8 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}
entries := make([]CorrectionLedgerEntry, 0)
seen := make(map[string]bool)
identity := func(instance string, index int) string { return instance + "\x00" + fmt.Sprint(index) }
for _, module := range runOutput.ModuleResults {
decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord)
for _, decision := range module.ValidatorDecisions {
@@ -62,6 +66,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}
for _, change := range module.AppliedChanges {
seen[identity(module.ModuleInstance, change.ProposalIndex)] = true
entries = append(entries, CorrectionLedgerEntry{
RunID: runID,
ModuleKey: module.ModuleKey,
@@ -78,6 +83,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
for _, change := range module.SkippedChanges {
seen[identity(module.ModuleInstance, change.ProposalIndex)] = true
entries = append(entries, CorrectionLedgerEntry{
RunID: runID,
ModuleKey: module.ModuleKey,
@@ -95,6 +101,7 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
for _, rejection := range module.ValidatorRejected {
seen[identity(module.ModuleInstance, rejection.ProposalIndex)] = true
entries = append(entries, CorrectionLedgerEntry{
RunID: runID,
ModuleKey: module.ModuleKey,
@@ -122,6 +129,34 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
}
for _, candidate := range runOutput.Candidates {
if seen[identity(candidate.ModuleInstance, candidate.ProposalIndex)] {
continue
}
var decisions []runner.ValidatorDecisionRecord
policy := ""
for _, module := range runOutput.ModuleResults {
if module.ModuleInstance != candidate.ModuleInstance {
continue
}
policy = string(module.ReplacementPolicy)
for _, decision := range module.ValidatorDecisions {
if decision.ProposalIndex == candidate.ProposalIndex {
decisions = append(decisions, decision)
}
}
break
}
entries = append(entries, CorrectionLedgerEntry{
RunID: runID, ModuleKey: candidate.ModuleKey, ModuleInstance: candidate.ModuleInstance,
ProposalIndex: candidate.ProposalIndex, SegmentID: candidate.TargetSegmentID,
OriginalText: candidate.OriginalText, ProposedCorrectedText: candidate.CorrectedText,
ReplacementPolicy: policy, Disposition: correctionDispositionInterrupted,
DispositionReasonCode: "not_applied", DispositionMessage: "candidate did not reach application",
DeterministicValidatorResults: filterLedgerDecisions(decisions, false),
LLMValidatorResults: filterLedgerDecisions(decisions, true),
})
}
sort.SliceStable(entries, func(i, j int) bool {
if entries[i].ModuleInstance != entries[j].ModuleInstance {

22
internal/framework/processreport/correction_ledger_test.go
View File

@@ -126,3 +126,25 @@ func TestHasSkippedCorrectionsIncludesApplicationSkipsAndValidatorRejections(t *
t.Fatal("validator rejections should count as skipped corrections")
}
}
func TestCorrectionLedgerIncludesUnappliedCandidateAfterFatalValidation(t *testing.T) {
output := &runner.RunOutput{
Candidates: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 7, OriginalText: "old", CorrectedText: "new"}, ProposalMetadata: proposals.ProposalMetadata{ModuleKey: "grammar", ModuleInstance: "grammar", ProposalIndex: 2}}},
ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar", Status: runner.ModuleStatusFailed,
ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "spoken_form_plausibility", ProposalIndex: 2, Approved: false, ReasonCode: "transport_failed"}}}},
}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
if len(ledger) != 2 {
t.Fatalf("missing candidate: %+v", ledger)
}
for _, entry := range ledger {
if entry.ProposalIndex != 2 {
continue
}
if entry.Disposition != correctionDispositionInterrupted || entry.OriginalText != "old" || len(entry.LLMValidatorResults) != 1 {
t.Fatalf("candidate evidence lost: %+v", entry)
}
return
}
t.Fatal("candidate missing")
}

28
internal/framework/runner/runner.go
View File

@@ -94,20 +94,22 @@ type RunInput struct {
ProposalDiagnosticsDir string
ValidationLLMClient contracts.StructuredLLMClient
ValidationDiagnosticsDir string
CaptureCandidates bool
timings *validatorTimingCollector
}
// RunOutput is the deterministic runner output.
type RunOutput struct {
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Observations llm.ObservationSnapshot `json:"-"`
ObservationsAvailable bool `json:"-"`
LLMSummary llm.ExecutionSummary `json:"-"`
RunWallTimeMS int64 `json:"-"`
ValidatorTimings []ValidatorTiming `json:"-"`
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
Observations llm.ObservationSnapshot `json:"-"`
ObservationsAvailable bool `json:"-"`
LLMSummary llm.ExecutionSummary `json:"-"`
RunWallTimeMS int64 `json:"-"`
ValidatorTimings []ValidatorTiming `json:"-"`
}
func New(factory ModuleFactory) *Runner {
@@ -136,6 +138,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
}
inventory := make([]contracts.SectionMetadata, len(sections))
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections))
var candidates []proposals.EnrichedCorrectionProposal
for i, section := range sections {
inventory[i] = contracts.SectionMetadataFromSection(section)
}
@@ -151,7 +154,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
snapshot := func() RunOutput {
return RunOutput{
FinalTranscript: transcriptFromSections(sections), ModuleResults: results,
Sections: inventory, WorkUnits: workUnits,
Sections: inventory, WorkUnits: workUnits, Candidates: candidates,
}
}
@@ -185,6 +188,9 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.timings,
})
if input.CaptureCandidates {
candidates = append(candidates, pipelineResult.Candidates...)
}
if pipelineErr != nil {
failed := ModuleResult{
ModuleKey: spec.ModuleKey,
@@ -321,6 +327,7 @@ type sectionValidationResult struct {
type modulePipelineResult struct {
ProposalCount int
Candidates []proposals.EnrichedCorrectionProposal
Approved []proposals.EnrichedCorrectionProposal
Operations []outcomes.Operation
WorkUnits []WorkUnitOutcome
@@ -473,6 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[nextSectionToProcess].Operations {
operation := &out.WorkUnits[nextSectionToProcess].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {

25
internal/framework/runner/runner_test.go
View File

@@ -128,6 +128,31 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
}
}
func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good", Confidence: 1}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
input := RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "bad"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}}
without, err := r.Run(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(without.Candidates) != 0 {
t.Fatalf("debug-off candidate capture: %+v", without.Candidates)
}
input.CaptureCandidates = true
with, err := r.Run(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(with.Candidates) != 1 || with.Candidates[0].ProposalIndex != 0 || len(with.ModuleResults[0].ValidatorRejected) != 1 {
t.Fatalf("filtered candidate missing from debug evidence: %+v", with)
}
}
func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return m.proposeResult(req)
}