eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Make debug evidence opt-in and retain all candidates

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:41:10 +00:00
parent a80eed4745
commit 87b8d9ad5c
10 changed files with 246 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
internal/framework/runner/runner.go
View File

@@ -94,20 +94,22 @@ type RunInput struct {
ProposalDiagnosticsDir string
ValidationLLMClient contracts.StructuredLLMClient
ValidationDiagnosticsDir string
CaptureCandidates bool
timings *validatorTimingCollector
}
// RunOutput is the deterministic runner output.
type RunOutput struct {
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Observations llm.ObservationSnapshot `json:"-"`
ObservationsAvailable bool `json:"-"`
LLMSummary llm.ExecutionSummary `json:"-"`
RunWallTimeMS int64 `json:"-"`
ValidatorTimings []ValidatorTiming `json:"-"`
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
Observations llm.ObservationSnapshot `json:"-"`
ObservationsAvailable bool `json:"-"`
LLMSummary llm.ExecutionSummary `json:"-"`
RunWallTimeMS int64 `json:"-"`
ValidatorTimings []ValidatorTiming `json:"-"`
}
func New(factory ModuleFactory) *Runner {
@@ -136,6 +138,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
}
inventory := make([]contracts.SectionMetadata, len(sections))
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections))
var candidates []proposals.EnrichedCorrectionProposal
for i, section := range sections {
inventory[i] = contracts.SectionMetadataFromSection(section)
}
@@ -151,7 +154,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
snapshot := func() RunOutput {
return RunOutput{
FinalTranscript: transcriptFromSections(sections), ModuleResults: results,
Sections: inventory, WorkUnits: workUnits,
Sections: inventory, WorkUnits: workUnits, Candidates: candidates,
}
}
@@ -185,6 +188,9 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.timings,
})
if input.CaptureCandidates {
candidates = append(candidates, pipelineResult.Candidates...)
}
if pipelineErr != nil {
failed := ModuleResult{
ModuleKey: spec.ModuleKey,
@@ -321,6 +327,7 @@ type sectionValidationResult struct {
type modulePipelineResult struct {
ProposalCount int
Candidates []proposals.EnrichedCorrectionProposal
Approved []proposals.EnrichedCorrectionProposal
Operations []outcomes.Operation
WorkUnits []WorkUnitOutcome
@@ -473,6 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[nextSectionToProcess].Operations {
operation := &out.WorkUnits[nextSectionToProcess].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {

25
internal/framework/runner/runner_test.go
View File

@@ -128,6 +128,31 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
}
}
func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good", Confidence: 1}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
input := RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "bad"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}}
without, err := r.Run(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(without.Candidates) != 0 {
t.Fatalf("debug-off candidate capture: %+v", without.Candidates)
}
input.CaptureCandidates = true
with, err := r.Run(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(with.Candidates) != 1 || with.Candidates[0].ProposalIndex != 0 || len(with.ModuleResults[0].ValidatorRejected) != 1 {
t.Fatalf("filtered candidate missing from debug evidence: %+v", with)
}
}
func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return m.proposeResult(req)
}