Retain processing evidence only for selected outputs

This commit is contained in:
2026-11-16 15:08:02 +00:00
parent 5167154a1c
commit 891d4aac2c
21 changed files with 494 additions and 229 deletions

View File

@@ -79,3 +79,9 @@ own phase and available operation coordinates, and peer interruptions and later
file failures preserve their own attribution. The correction ledger joins one
candidate row by module instance and proposal index, preserving repair lineage
and the recorded validator execution class.
The invocation selects retained evidence once. Summary-only processing
accumulates counters as completed sections leave the work window. Ordinary
reports retain the full work matrix and operation facts. Debug retains
candidate and decision history, repair lineage, and native observations.
Without debug, the runtime skips prepared message snapshots.

View File

@@ -658,6 +658,22 @@ Audit #32 and supporting work for #30, #33, and #44.
Resource measurements support reduction of waste without setting a
constant-memory or disk-use requirement.
**Complete.** The CLI selects ordinary report and debug evidence before
processing. Section outcomes feed shared totals as they drain. The runner
retains the full work grid only for reports, and retains candidate and
validator history only for debug in the CLI. Duplicate module operations and
unused proposal totals were removed. Prepared messages are captured only for
interaction diagnostics. Mode comparison tests and the full offline suite
passed.
A synthetic 4,000-section run with recoverable failures, proposal recoveries,
and apply-time filtering retained about 0.46 MB with summary only, 2.73 MB
with ordinary reports, 1.56 MB with debug evidence, and 3.83 MB with both
after garbage collection. Allocations per run were approximately 20.9,
22.0, 24.0, and 25.0 MB respectively. These runner measurements exclude
provider calls and disk artifacts. All four modes agreed on the transcript
and counters. Temporary measurement instrumentation was removed.
## Stage 19 — Emit native processing progress and update operator guidance
Audit #5, #57, and the operator-facing parts of #8, #24 and #25.

View File

@@ -19,6 +19,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
cfg := &loaded.Configuration.Config
specs := prepared.Prepared.Specs
debug := prepared.Prepared.Debug
evidence := runner.EvidenceSelection{Reports: cfg.Reporting.Enabled, Debug: debug != nil}
var observer *llm.ObservationCollector
var hooks llm.CallHooks
if debug != nil {
@@ -57,7 +58,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
}
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
Interactions: interactions, CaptureCandidates: debug != nil,
Interactions: interactions, Evidence: &evidence,
})
if debug != nil {
for _, warning := range writer.Warnings() {
@@ -68,6 +69,6 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
}
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started,
ModuleSpecs: specs, RunOutput: &output, ProcessingError: runErr, SummaryOnly: !cfg.Reporting.Enabled})
ModuleSpecs: specs, RunOutput: &output, ProcessingError: runErr, SummaryOnly: !evidence.Reports})
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
}

View File

@@ -64,6 +64,7 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
var terminal pk.ExecutionEvent
var terminalAvailable bool
var prepared *structuredoutput.PreparedSnapshot
var prompt structuredoutput.PromptMetadata
var record structuredoutput.CallRecord
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
record.CallID = c.runtime.nextCallID.Add(1)
@@ -90,19 +91,19 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
response.Execution.HTTPStatus = &status
}
}
if prepared != nil {
response.Prepared = prepared
response.Prompt = prepared.Prompt
response.Model = prepared.Prompt.Model
}
response.Prepared = prepared
response.Prompt = prompt
response.Model = prompt.Model
mu.Unlock()
if result != nil {
response.FinalContent = result.RawOutput
if req.CaptureDiagnostics {
response.FinalContent = result.RawOutput
}
response.Model = result.ModelName
}
err := c.finishResponse(parent, &response, result, runErr, out)
if onCall := c.hooks.OnCall; onCall != nil {
if response.Prepared != nil {
if response.Prompt.PromptID != "" {
record.Prompt = response.Prompt
}
record.Finished = time.Now().UTC()
@@ -146,12 +147,18 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
}
}),
OnPrepared: func(_ context.Context, details pk.PreparedRun) {
snapshot := preparedSnapshot(details)
}
if req.CaptureDiagnostics || c.hooks.OnCall != nil {
call.request.OnPrepared = func(_ context.Context, details pk.PreparedRun) {
metadata := promptMetadata(details)
var snapshot *structuredoutput.PreparedSnapshot
if req.CaptureDiagnostics {
snapshot = &structuredoutput.PreparedSnapshot{Messages: convertMessages(details.Messages), Prompt: metadata}
}
mu.Lock()
prepared = snapshot
prepared, prompt = snapshot, metadata
mu.Unlock()
},
}
}
return call, nil
}
@@ -290,11 +297,6 @@ func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
}
}
func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot {
messages := convertMessages(details.Messages)
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
}
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
result := make([]structuredoutput.Message, len(messages))
for i, message := range messages {

View File

@@ -89,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
id = prompts.PromptIDValidatorEditorialReview
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
}
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
return structuredoutput.Request{CaptureDiagnostics: true, PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
}
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {

View File

@@ -86,7 +86,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
if turns == 2 {
return nil, nil
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
})
if err != nil || exhausted || turns != 2 {
t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err)
@@ -111,7 +111,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
if turn.Response != nil {
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
}
_, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent)
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
@@ -172,7 +172,7 @@ func TestCoordinatedRepairUsesReplacementAndCountsStructuralRepair(t *testing.T)
result = turn.Response
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
})
if err != nil || exhausted || result == nil {
t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err)

View File

@@ -55,7 +55,7 @@ func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
return nil, nil
}
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
@@ -102,7 +102,7 @@ func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
}
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
if !errors.Is(err, context.Canceled) || deliveries != 1 {
t.Fatalf("deliveries=%d err=%v", deliveries, err)
@@ -132,7 +132,7 @@ func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
return nil, err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
summary := Summarize(collector.Snapshot())
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {

View File

@@ -109,6 +109,7 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
var result map[string]any
capturing := request
capturing.CaptureExchange = true
capturing.CaptureDiagnostics = true
response, err := client.CompleteStructured(context.Background(), capturing, &result)
if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) {
t.Fatalf("captured final response: %+v", response.Exchange)
@@ -261,6 +262,7 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
if err != nil {
t.Fatal(err)
}
req.CaptureDiagnostics = true
response, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
if err != nil {
t.Fatal(err)

View File

@@ -121,3 +121,57 @@ func ExecutionRecoveries(counts ExecutionCounts) []RecoveryReason {
}
return reasons
}
// WorkTotals retains the facts shared by the CLI summary and ordinary reports.
type WorkTotals struct {
Total int
Started int
Completed int
WithErrors int
Corrections int
ModelOperations int
ProposalResponses int
Recoveries int
FilteringResponses int
}
func (t *WorkTotals) Record(state WorkState, operations []Operation) {
if state != WorkNotStarted {
t.Started++
}
if state == WorkCompleted {
t.Completed++
}
hasError := false
for _, operation := range operations {
if operation.Reference.Kind != OperationValidationInput {
t.ModelOperations++
}
if operation.Succeeded {
if operation.Reference.Kind == OperationProposal {
t.ProposalResponses++
if len(operation.Filtered) > 0 {
t.FilteringResponses++
}
}
if len(operation.RecoveryReasons) > 0 {
t.Recoveries++
}
}
hasError = hasError || operation.FailureClass == FailureRecoverable
}
if hasError {
t.WithErrors++
}
}
func (t *WorkTotals) Add(module WorkTotals) {
t.Started += module.Started
t.Completed += module.Completed
t.WithErrors += module.WithErrors
t.Corrections += module.Corrections
t.ModelOperations += module.ModelOperations
t.ProposalResponses += module.ProposalResponses
t.Recoveries += module.Recoveries
t.FilteringResponses += module.FilteringResponses
}

View File

@@ -74,7 +74,7 @@ func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) {
cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}}
result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, CaptureCandidates: true,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, Evidence: &runner.EvidenceSelection{Reports: true, Debug: true},
})
if !errors.Is(err, origin) || client.calls != 3 {
t.Fatalf("fatal cause/calls: %v %d", err, client.calls)

View File

@@ -66,59 +66,35 @@ func Build(input BuildInput) reporting.Snapshot {
return units[i].Section.Index < units[j].Section.Index
})
}
total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0
corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0
var measured outcomes.WorkTotals
if input.RunOutput.Totals != nil {
measured = *input.RunOutput.Totals
} else {
measured.Total = len(units)
for _, unit := range units {
measured.Record(unit.State, unit.Operations)
measured.Corrections += unit.CorrectionsApplied
}
}
for _, unit := range units {
if unit.State == outcomes.WorkNotStarted {
notStarted++
} else {
started++
}
if unit.State == outcomes.WorkCompleted {
completed++
}
if unit.State == outcomes.WorkInterrupted {
interrupted++
}
corrections += unit.CorrectionsApplied
operations := unit.Operations
if details {
operations = append([]outcomes.Operation(nil), operations...)
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
if !details {
break
}
operations := append([]outcomes.Operation(nil), unit.Operations...)
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
var failures []reporting.Failure
unitHasError := false
unitRecoveries := 0
for _, operation := range operations {
if operation.Reference.Kind != outcomes.OperationValidationInput {
llmOperations++
}
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
proposalResponses++
}
if operation.Succeeded && len(operation.RecoveryReasons) > 0 {
recoveryCount++
unitRecoveries++
if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal {
filteringResponses++
}
if details {
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
}
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
}
if operation.FailureClass == outcomes.FailureRecoverable {
unitHasError = true
if details {
failures = append(failures, mapFailure(operation))
}
failures = append(failures, mapFailure(operation))
}
}
if unitHasError {
withErrors++
}
if !details {
continue
}
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied, RecoveryCount: unitRecoveries}
@@ -153,12 +129,13 @@ func Build(input BuildInput) reporting.Snapshot {
return a.ProposalIndex < b.ProposalIndex
})
runErrors := len(failures)
interrupted, notStarted := measured.Started-measured.Completed, measured.Total-measured.Started
snapshot.Summary = reporting.Summary{
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: &notStarted, WorkUnitsWithErrors: &withErrors,
RunErrors: &runErrors, CorrectionsApplied: &corrections, LLMOperations: &llmOperations,
ProposalResponses: &proposalResponses, Recoveries: &recoveryCount,
ProposalResponsesWithFiltering: &filteringResponses,
WorkUnitsTotal: &measured.Total, WorkUnitsStarted: &measured.Started, WorkUnitsCompleted: &measured.Completed,
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: &notStarted, WorkUnitsWithErrors: &measured.WithErrors,
RunErrors: &runErrors, CorrectionsApplied: &measured.Corrections, LLMOperations: &measured.ModelOperations,
ProposalResponses: &measured.ProposalResponses, Recoveries: &measured.Recoveries,
ProposalResponsesWithFiltering: &measured.FilteringResponses,
}
} else if len(failures) > 0 {
runErrors := len(failures)

View File

@@ -82,9 +82,10 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
)
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
CaptureExchange: req.CaptureExchange,
PromptID: req.PromptID,
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
CaptureExchange: req.CaptureExchange,
CaptureDiagnostics: req.Interactions != nil,
PromptID: req.PromptID,
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
Context: coordinates,
}, &response)
sectionIndex := 0

View File

@@ -207,7 +207,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, CaptureCandidates: true,
Config: &cfg, Evidence: &EvidenceSelection{Reports: true, Debug: true},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
})

View File

@@ -0,0 +1,94 @@
package runner
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
func evidenceFixture(size int) (*Runner, RunInput) {
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: make([]schema.Segment, size)}
for i := range transcript.Segments {
transcript.Segments[i] = schema.Segment{ID: i + 1, Text: "word sample"}
}
validator := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, candidate := range req.CandidateProposal {
decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
}
return validators.Result{Decisions: decisions}, nil
}}
module := outcomeModule{
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
operation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}}
category := req.Section.Index % 4
if category == 0 {
operation.FailureClass, operation.FailureReason = outcomes.FailureRecoverable, outcomes.ReasonTransportFailed
return contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, nil
}
operation.Succeeded = true
original := "word"
if category == 1 {
original = "absent"
}
if category == 2 {
operation.RecoveryReasons = []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered}
}
return contracts.ProposalResult{
Operations: []outcomes.Operation{operation},
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: req.Section.StartSegmentID, OriginalText: original, CorrectedText: "Word"}},
}, nil
},
}
return New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}), RunInput{
Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
}
}
func TestEvidenceSelectionKeepsTotalsAndRequestedRecords(t *testing.T) {
r, input := evidenceFixture(40)
want := outcomes.WorkTotals{Total: 40, Started: 40, Completed: 40, WithErrors: 10, Corrections: 20,
ModelOperations: 40, ProposalResponses: 30, Recoveries: 20, FilteringResponses: 10}
var baseline *schema.Transcript
for _, tc := range []struct {
name string
selection EvidenceSelection
units int
candidates int
}{
{name: "summary"},
{name: "reports", selection: EvidenceSelection{Reports: true}, units: 40},
{name: "debug", selection: EvidenceSelection{Debug: true}, candidates: 30},
{name: "reports and debug", selection: EvidenceSelection{Reports: true, Debug: true}, units: 40, candidates: 30},
} {
t.Run(tc.name, func(t *testing.T) {
input.Evidence = &tc.selection
out, err := r.Run(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if out.Totals == nil || *out.Totals != want {
t.Fatalf("totals: got %+v, want %+v", out.Totals, want)
}
if len(out.WorkUnits) != tc.units || len(out.Candidates) != tc.candidates ||
len(out.ModuleResults[0].ValidatorDecisions) != tc.candidates {
t.Fatalf("retained units=%d candidates=%d decisions=%d", len(out.WorkUnits), len(out.Candidates), len(out.ModuleResults[0].ValidatorDecisions))
}
if baseline == nil {
baseline = out.FinalTranscript
} else if !reflect.DeepEqual(baseline, out.FinalTranscript) {
t.Fatal("output selection changed final corrections")
}
})
}
}

View File

@@ -161,7 +161,7 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
}
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, CaptureDiagnostics: debug, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
}
exhausted, coordinationErr := input.Spec.RepairCoordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response == nil {

View File

@@ -45,8 +45,6 @@ type ModuleResult struct {
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Status string `json:"status"`
ProposalCount int `json:"proposal_count"`
Operations []outcomes.Operation `json:"operations,omitempty"`
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
@@ -95,13 +93,32 @@ type ValidatorRejectedChange struct {
Message string `json:"message"`
}
// EvidenceSelection selects the detailed views consumed by the invocation.
type EvidenceSelection struct {
Reports bool
Debug bool
}
type evidenceNeeds struct {
reports bool
decisions bool
candidates bool
}
func selectEvidence(selection *EvidenceSelection) evidenceNeeds {
if selection == nil {
return evidenceNeeds{reports: true, decisions: true}
}
return evidenceNeeds{reports: selection.Reports, decisions: selection.Debug, candidates: selection.Debug}
}
// RunInput is the deterministic runner input.
type RunInput struct {
Config *config.Config
Transcript *schema.Transcript
ModuleSpecs []contracts.ModuleRunSpec
Interactions structuredoutput.InteractionSink
CaptureCandidates bool
Config *config.Config
Transcript *schema.Transcript
ModuleSpecs []contracts.ModuleRunSpec
Interactions structuredoutput.InteractionSink
Evidence *EvidenceSelection
}
// RunOutput is the deterministic runner output.
@@ -109,6 +126,7 @@ type RunOutput struct {
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
Totals *outcomes.WorkTotals `json:"-"`
InventoryAvailable bool `json:"-"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
@@ -132,54 +150,70 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil {
return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)}
}
inventory := make([]contracts.SectionMetadata, len(sections))
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections))
evidence := selectEvidence(input.Evidence)
totals := outcomes.WorkTotals{Total: len(input.ModuleSpecs) * len(sections)}
var inventory []contracts.SectionMetadata
var workUnits []WorkUnitOutcome
var candidates []proposals.EnrichedCorrectionProposal
for i, section := range sections {
inventory[i] = contracts.SectionMetadataFromSection(section)
}
for moduleIndex, spec := range input.ModuleSpecs {
for _, section := range inventory {
workUnits = append(workUnits, WorkUnitOutcome{
ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, Section: section,
State: outcomes.WorkNotStarted,
})
if evidence.reports {
inventory = make([]contracts.SectionMetadata, len(sections))
for i, section := range sections {
inventory[i] = contracts.SectionMetadataFromSection(section)
}
workUnits = make([]WorkUnitOutcome, 0, totals.Total)
for moduleIndex, spec := range input.ModuleSpecs {
for _, section := range inventory {
workUnits = append(workUnits, newWorkUnit(spec, section, moduleIndex))
}
}
}
snapshot := func() RunOutput {
return RunOutput{
FinalTranscript: transcriptFromSections(sections), ModuleResults: results,
Sections: inventory, InventoryAvailable: true, WorkUnits: workUnits, Candidates: candidates,
Sections: inventory, InventoryAvailable: true, Totals: &totals, WorkUnits: workUnits, Candidates: candidates,
}
}
for moduleIndex, spec := range input.ModuleSpecs {
execution, err := r.executeModule(ctx, spec, sections, input.Interactions)
units := workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
copy(units, execution.pipeline.WorkUnits)
if err == nil && ctx.Err() != nil {
err = locateFailure(ctx.Err(), "application", failureReference(spec, nil, ""), false, nil)
var reportUnits []WorkUnitOutcome
if evidence.reports {
reportUnits = workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
}
execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits)
units := execution.pipeline.WorkUnits
if err == nil {
previous := sections
sections = execution.updated
attributeApplyFiltering(previous, execution.pipeline.Approved, execution.skipped, execution.policy, units)
filters := findApplyFiltering(sections, execution.pipeline.Approved, execution.skipped, execution.policy)
if canceled := ctx.Err(); canceled != nil {
err = locateFailure(canceled, "application", failureReference(spec, nil, ""), false, nil)
} else {
execution.pipeline.recordApplyFiltering(filters, evidence.reports)
sections = execution.updated
execution.pipeline.Totals.Started = len(sections)
execution.pipeline.Totals.Completed = len(sections)
execution.pipeline.Totals.Corrections = len(execution.applied)
}
}
var failure *ExecutionFailure
applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application"
appliedIndices := make(map[int]struct{}, len(execution.applied))
for _, change := range execution.applied {
appliedIndices[change.ProposalIndex] = struct{}{}
if applicationInterrupted {
execution.pipeline.Totals.Completed = 0
}
totals.Add(execution.pipeline.Totals)
appliedIndices := make(map[int]struct{})
if evidence.reports || evidence.candidates {
for _, change := range execution.applied {
appliedIndices[change.ProposalIndex] = struct{}{}
}
}
var operations []outcomes.Operation
for i := range units {
unit := &units[i]
unit.ModuleIndex = moduleIndex
if err == nil {
unit.State = outcomes.WorkCompleted
unit.CorrectionsApplied = execution.appliedBySection[i]
if evidence.reports {
unit.CorrectionsApplied = execution.appliedBySection[unit.Section.Index]
}
} else if applicationInterrupted && unit.State != outcomes.WorkNotStarted {
unit.State = outcomes.WorkInterrupted
}
@@ -196,16 +230,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
branch.Outcome = "approved_skipped"
}
}
operations = append(operations, unit.Operations...)
}
if input.CaptureCandidates {
candidates = append(candidates, execution.pipeline.Candidates...)
if !evidence.reports && evidence.candidates {
workUnits = append(workUnits, units...)
}
candidates = append(candidates, execution.pipeline.Candidates...)
result := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy,
ProposalCount: execution.pipeline.ProposalCount, Operations: operations,
ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected,
}
if err != nil {
@@ -213,8 +245,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
result.ErrorMessage = err.Error()
} else {
result.Status = ModuleStatusSuccess
result.AppliedChanges = execution.applied
result.SkippedChanges = execution.skipped
if evidence.decisions {
result.AppliedChanges = execution.applied
result.SkippedChanges = execution.skipped
}
}
results = append(results, result)
if err != nil {
@@ -237,7 +271,7 @@ type moduleExecution struct {
appliedBySection []int
}
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink) (moduleExecution, error) {
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome) (moduleExecution, error) {
var execution moduleExecution
origin := failureReference(spec, nil, "")
if err := ctx.Err(); err != nil {
@@ -253,6 +287,8 @@ func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec
}
execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{
Module: module,
Evidence: &evidence,
ReportUnits: reportUnits,
Spec: spec,
Policy: execution.policy,
Sections: sections,
@@ -323,6 +359,8 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
}
type collectSectionProposalsInput struct {
Evidence *evidenceNeeds
ReportUnits []WorkUnitOutcome
Interactions structuredoutput.InteractionSink
Module contracts.TranscriptModule
Spec contracts.ModuleRunSpec
@@ -356,16 +394,18 @@ type sectionValidationResult struct {
}
type sectionWork struct {
unit WorkUnitOutcome
candidates []proposals.EnrichedCorrectionProposal
validation *sectionValidationResult
finished bool
}
type modulePipelineResult struct {
ProposalCount int
Totals outcomes.WorkTotals
proposalFacts map[int]proposalRecovery
revised map[int]struct{}
Candidates []proposals.EnrichedCorrectionProposal
Approved []proposals.EnrichedCorrectionProposal
Operations []outcomes.Operation
WorkUnits []WorkUnitOutcome
ValidatorDecisions []ValidatorDecisionRecord
ValidatorRejected []ValidatorRejectedChange
@@ -410,17 +450,16 @@ func proposeSection(ctx context.Context, input collectSectionProposalsInput, sec
}
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
out := modulePipelineResult{
Approved: make([]proposals.EnrichedCorrectionProposal, 0),
Operations: make([]outcomes.Operation, 0),
WorkUnits: make([]WorkUnitOutcome, len(input.Sections)),
ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
ValidatorRejected: make([]ValidatorRejectedChange, 0),
var out modulePipelineResult
evidence := selectEvidence(nil)
if input.Evidence != nil {
evidence = *input.Evidence
}
for i, section := range input.Sections {
out.WorkUnits[i] = WorkUnitOutcome{
ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, StageIndex: input.Spec.Index, StageID: input.Spec.ID,
Section: contracts.SectionMetadataFromSection(section), State: outcomes.WorkNotStarted,
out.WorkUnits = input.ReportUnits
if evidence.reports && out.WorkUnits == nil {
out.WorkUnits = make([]WorkUnitOutcome, len(input.Sections))
for i, section := range input.Sections {
out.WorkUnits[i] = newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(section), 0)
}
}
@@ -487,17 +526,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
return sectionEnriched
}
recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) {
unit := &out.WorkUnits[sectionPos]
recordCandidates := func(unit *WorkUnitOutcome, enriched []proposals.EnrichedCorrectionProposal) {
unit.ProposalIndexOffset = nextProposalIndex
for _, candidate := range enriched {
candidate.ProposalIndex += nextProposalIndex
out.Candidates = append(out.Candidates, candidate)
if evidence.candidates {
for _, candidate := range enriched {
candidate.ProposalIndex += nextProposalIndex
out.Candidates = append(out.Candidates, candidate)
}
}
out.ProposalCount += len(enriched)
for operationIndex := range unit.Operations {
operation := &unit.Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
if evidence.reports && operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range enriched {
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
}
@@ -516,16 +555,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
interactions = capture
}
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: interactions,
Spec: input.Spec,
SuppressDecisionHistory: !evidence.decisions,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
@@ -555,8 +595,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
failure: failure,
}
}
recordValidation := func(res sectionValidationResult) {
unit := &out.WorkUnits[res.sectionPos]
recordValidation := func(unit *WorkUnitOutcome, res sectionValidationResult) {
unit.Operations = append(unit.Operations, res.operations...)
unit.Repairs = res.repairs
mergeFilteredProposals(unit, res.filtered)
@@ -567,18 +606,30 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
unit.State = outcomes.WorkInterrupted
}
out.Approved = append(out.Approved, res.approved...)
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
if evidence.decisions {
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
}
if len(res.approved) > 0 {
out.rememberProposalRecovery(unit)
}
}
nextAdmission, active := 0, 0
for {
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
offset := nextProposalIndex
recordCandidates(nextSectionToRecord, work.candidates)
recordCandidates(&work.unit, work.candidates)
if work.validation != nil {
work.validation.assignProposalIndices(offset)
recordValidation(*work.validation)
recordValidation(&work.unit, *work.validation)
}
out.Totals.Record(work.unit.State, work.unit.Operations)
if evidence.reports {
out.WorkUnits[nextSectionToRecord] = work.unit
} else if evidence.candidates && len(work.unit.Repairs) > 0 {
work.unit.Operations = nil
out.WorkUnits = append(out.WorkUnits, work.unit)
}
delete(pending, nextSectionToRecord)
nextSectionToRecord++
@@ -586,7 +637,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
sectionPos := nextAdmission
nextAdmission++
pending[sectionPos] = &sectionWork{}
pending[sectionPos] = &sectionWork{unit: newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(input.Sections[sectionPos]), 0)}
active++
workers.Go(func() {
proposed := proposeSection(runCtx, input, sectionPos)
@@ -600,13 +651,13 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
select {
case result := <-proposalResults:
active--
unit := &out.WorkUnits[result.sectionPos]
work := pending[result.sectionPos]
unit := &work.unit
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.started {
unit.State = outcomes.WorkInterrupted
}
unit.Failure = result.failure
work := pending[result.sectionPos]
if result.started && result.failure == nil {
work.candidates = makeCandidates(result)
if runCtx.Err() == nil {
@@ -633,10 +684,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
})
for _, unit := range out.WorkUnits {
out.Operations = append(out.Operations, unit.Operations...)
}
sort.SliceStable(out.ValidatorDecisions, func(i, j int) bool {
if out.ValidatorDecisions[i].ProposalIndex != out.ValidatorDecisions[j].ProposalIndex {
return out.ValidatorDecisions[i].ProposalIndex < out.ValidatorDecisions[j].ProposalIndex
@@ -668,19 +715,20 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
type validateSectionCandidatesInput struct {
Section *validators.SectionLookup
RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []validators.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
GlossaryView *validators.GlossaryView
SuppressDecisionHistory bool
Section *validators.SectionLookup
RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []validators.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
GlossaryView *validators.GlossaryView
}
type validateSectionCandidatesResult struct {
@@ -781,16 +829,18 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
noEffect[d.ProposalIndex] = true
}
}
decisions = append(decisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ExecutionClass: class,
ProposalIndex: d.ProposalIndex,
Approved: approved,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
if !input.SuppressDecisionHistory {
decisions = append(decisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ExecutionClass: class,
ProposalIndex: d.ProposalIndex,
Approved: approved,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
}
if approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
continue
@@ -800,19 +850,21 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
}
}
p := byIndex[d.ProposalIndex]
rejected = append(rejected, ValidatorRejectedChange{
ValidatorName: validator.Name(),
ProposalIndex: p.ProposalIndex,
ModuleKey: p.ModuleKey,
ModuleInstance: p.ModuleInstance,
TargetSegmentID: p.TargetSegmentID,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
if feedback || !input.SuppressDecisionHistory {
p := byIndex[d.ProposalIndex]
rejected = append(rejected, ValidatorRejectedChange{
ValidatorName: validator.Name(),
ProposalIndex: p.ProposalIndex,
ModuleKey: p.ModuleKey,
ModuleInstance: p.ModuleInstance,
TargetSegmentID: p.TargetSegmentID,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
}
}
if err != nil {
@@ -877,9 +929,9 @@ func mergeOperationFiltering(operation *outcomes.Operation, filtered []outcomes.
}
}
func attributeApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy, units []WorkUnitOutcome) {
func findApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy) map[int][]outcomes.FilteredProposal {
if len(skipped) == 0 {
return
return nil
}
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved))
sectionPosition := make(map[int]int, len(sections))
@@ -916,26 +968,7 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
}
filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1})
}
for pos, items := range filtered {
unit := &units[pos]
revisions := make(map[int]*outcomes.Operation)
for i := range unit.Operations {
op := &unit.Operations[i]
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Succeeded && op.Reference.ProposalIndex != nil &&
slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
revisions[*op.Reference.ProposalIndex] = op
}
}
original := items[:0]
for _, item := range items {
if op := revisions[item.ProposalIndex]; op != nil {
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
} else {
original = append(original, item)
}
}
mergeFilteredProposals(unit, original)
}
return filtered
}
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {

View File

@@ -173,7 +173,7 @@ func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
if len(without.Candidates) != 0 {
t.Fatalf("debug-off candidate capture: %+v", without.Candidates)
}
input.CaptureCandidates = true
input.Evidence = &EvidenceSelection{Reports: true, Debug: true}
with, err := r.Run(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -844,7 +844,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
},
},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, CaptureCandidates: true})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, Evidence: &EvidenceSelection{Reports: true, Debug: true}})
if err != nil {
t.Fatal(err)
}
@@ -973,8 +973,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
}
if len(out.ModuleResults[1].Operations) != 1 || out.ModuleResults[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid {
t.Fatalf("expected invalid decision-set operation, got %+v", out.ModuleResults[1].Operations)
if len(out.WorkUnits[1].Operations) != 1 || out.WorkUnits[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid {
t.Fatalf("expected invalid decision-set operation, got %+v", out.WorkUnits[1].Operations)
}
}

View File

@@ -218,7 +218,7 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true,
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), Evidence: &EvidenceSelection{Reports: true, Debug: true},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,

View File

@@ -0,0 +1,78 @@
package runner
import (
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
)
func newWorkUnit(spec contracts.ModuleRunSpec, section contracts.SectionMetadata, moduleIndex int) WorkUnitOutcome {
return WorkUnitOutcome{
ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, Section: section, State: outcomes.WorkNotStarted,
}
}
type proposalRecovery struct {
recovered bool
filtered bool
}
// Application can discover additional defects. Keep only the proposal operation
// flags needed to update totals and the identities of approved revisions.
func (p *modulePipelineResult) rememberProposalRecovery(unit *WorkUnitOutcome) {
for _, operation := range unit.Operations {
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
if p.proposalFacts == nil {
p.proposalFacts = make(map[int]proposalRecovery)
}
p.proposalFacts[unit.Section.Index] = proposalRecovery{
recovered: len(operation.RecoveryReasons) > 0, filtered: len(operation.Filtered) > 0,
}
break
}
}
for _, repair := range unit.Repairs {
if repair.Outcome == "approved" {
if p.revised == nil {
p.revised = make(map[int]struct{})
}
p.revised[repair.ProposalIndex] = struct{}{}
}
}
}
func (p *modulePipelineResult) recordApplyFiltering(filters map[int][]outcomes.FilteredProposal, reports bool) {
for pos, items := range filters {
original := items[:0]
for _, item := range items {
if _, repaired := p.revised[item.ProposalIndex]; !repaired {
original = append(original, item)
continue
}
if reports {
operations := p.WorkUnits[pos].Operations
for i := range operations {
op := &operations[i]
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Reference.ProposalIndex != nil &&
*op.Reference.ProposalIndex == item.ProposalIndex && slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
break
}
}
}
}
if facts, ok := p.proposalFacts[pos]; ok && len(original) != 0 {
if !facts.recovered {
p.Totals.Recoveries++
}
if !facts.filtered {
p.Totals.FilteringResponses++
}
if reports {
mergeFilteredProposals(&p.WorkUnits[pos], original)
}
}
}
}

View File

@@ -26,11 +26,12 @@ type DiagnosticContext struct {
// Request names an embedded prompt and response schema. Profile selection and
// credentials are supplied by the role-bound runtime client.
type Request struct {
PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"`
Schema responseschema.Key `json:"schema"`
Context DiagnosticContext `json:"context"`
CaptureExchange bool `json:"-"`
PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"`
Schema responseschema.Key `json:"schema"`
Context DiagnosticContext `json:"context"`
CaptureExchange bool `json:"-"`
CaptureDiagnostics bool `json:"-"`
}
// PromptMetadata is the execution snapshot returned by the runtime when available.

View File

@@ -130,7 +130,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: coordinates,
Context: coordinates, CaptureDiagnostics: req.Interactions != nil,
}, &response)
batchIndex := batch.BatchIndex
operation := outcomes.Operation{