eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Make debug evidence opt-in and retain all candidates

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:41:10 +00:00
parent a80eed4745
commit 87b8d9ad5c
10 changed files with 246 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
internal/framework/runner/runner.go
View File

@@ -94,20 +94,22 @@ type RunInput struct {
ProposalDiagnosticsDir string
ValidationLLMClient contracts.StructuredLLMClient
ValidationDiagnosticsDir string
CaptureCandidates bool
timings *validatorTimingCollector
}
// RunOutput is the deterministic runner output.
type RunOutput struct {
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Observations llm.ObservationSnapshot `json:"-"`
ObservationsAvailable bool `json:"-"`
LLMSummary llm.ExecutionSummary `json:"-"`
RunWallTimeMS int64 `json:"-"`
ValidatorTimings []ValidatorTiming `json:"-"`
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
Observations llm.ObservationSnapshot `json:"-"`
ObservationsAvailable bool `json:"-"`
LLMSummary llm.ExecutionSummary `json:"-"`
RunWallTimeMS int64 `json:"-"`
ValidatorTimings []ValidatorTiming `json:"-"`
}
func New(factory ModuleFactory) *Runner {
@@ -136,6 +138,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
}
inventory := make([]contracts.SectionMetadata, len(sections))
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections))
var candidates []proposals.EnrichedCorrectionProposal
for i, section := range sections {
inventory[i] = contracts.SectionMetadataFromSection(section)
}
@@ -151,7 +154,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
snapshot := func() RunOutput {
return RunOutput{
FinalTranscript: transcriptFromSections(sections), ModuleResults: results,
Sections: inventory, WorkUnits: workUnits,
Sections: inventory, WorkUnits: workUnits, Candidates: candidates,
}
}
@@ -185,6 +188,9 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.timings,
})
if input.CaptureCandidates {
candidates = append(candidates, pipelineResult.Candidates...)
}
if pipelineErr != nil {
failed := ModuleResult{
ModuleKey: spec.ModuleKey,
@@ -321,6 +327,7 @@ type sectionValidationResult struct {
type modulePipelineResult struct {
ProposalCount int
Candidates []proposals.EnrichedCorrectionProposal
Approved []proposals.EnrichedCorrectionProposal
Operations []outcomes.Operation
WorkUnits []WorkUnitOutcome
@@ -473,6 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
nextProposalIndex += len(sectionEnriched)
out.ProposalCount += len(sectionEnriched)
out.Candidates = append(out.Candidates, sectionEnriched...)
for operationIndex := range out.WorkUnits[nextSectionToProcess].Operations {
operation := &out.WorkUnits[nextSectionToProcess].Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {