eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Integrate partial processing outcomes and fatal cancellation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 17:23:59 +00:00
parent bb21a81805
commit b98604e5ca
4 changed files with 294 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
internal/framework/runner/runner_test.go
View File

@@ -16,6 +16,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
@@ -45,6 +46,92 @@ type fakeModule struct {
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
}
type outcomeModule struct {
fakeModule
proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error)
}
func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil
}}
second := fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"first": failed, "second": second}})
out, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {ModuleKey: "second", InstanceName: "second"}}})
if err != nil {
t.Fatal(err)
}
if out.FinalTranscript.Segments[0].Text != "the" || len(out.WorkUnits) != 2 || out.WorkUnits[0].State != outcomes.WorkCompleted || out.WorkUnits[1].CorrectionsApplied != 1 {
t.Fatalf("recoverable failure lost later work: %+v", out)
}
if len(out.WorkUnits[0].Operations) != 1 || out.WorkUnits[0].Operations[0].FailureReason != outcomes.ReasonTransportFailed {
t.Fatalf("missing proposal failure: %+v", out.WorkUnits[0])
}
}
func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) {
secondSaw := make([]int, 0)
partial := fakeValidator{name: "partial", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
if len(req.CandidateProposal) != 2 {
t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal))
}
return validators.Result{ValidatorName: "partial", Decisions: []validators.Decision{
{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: false, Disposition: validators.DispositionUnavailable, ReasonCode: string(outcomes.ReasonTransportFailed)},
{ProposalIndex: req.CandidateProposal[1].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved},
}, Operations: []outcomes.Operation{
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonTransportFailed},
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed},
}}, nil
}}
last := fakeValidator{name: "last", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
for _, candidate := range req.CandidateProposal {
secondSaw = append(secondSaw, candidate.ProposalIndex)
}
return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1},
}, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired}}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}})
out, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
if err != nil {
t.Fatal(err)
}
if out.FinalTranscript.Segments[0].Text != "teh dog" || len(secondSaw) != 1 || secondSaw[0] != 2 {
t.Fatalf("unchecked candidate applied or survivor lost: text=%q final=%v", out.FinalTranscript.Segments[0].Text, secondSaw)
}
unit := out.WorkUnits[0]
if unit.State != outcomes.WorkCompleted || unit.CorrectionsApplied != 1 || len(unit.Operations) != 3 {
t.Fatalf("expected one completed unit with two operation failures: %+v", unit)
}
proposal := unit.Operations[0]
if len(proposal.RecoveryReasons) != 2 || proposal.RecoveryReasons[0] != outcomes.RecoveryOutputRepaired || proposal.RecoveryReasons[1] != outcomes.RecoveryProposalsFiltered || len(proposal.Filtered) != 1 || proposal.Filtered[0].ProposalIndex != 0 {
t.Fatalf("repair and filtering must share one recovery: %+v", proposal)
}
}
func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
origin := errors.New("validator setup failed")
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []contracts.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 {
t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err)
}
}
func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return m.proposeResult(req)
}
func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
@@ -854,8 +941,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
}
if len(out.ModuleResults[1].Warnings) != 1 || out.ModuleResults[1].Warnings[0].ReasonCode != validators.ReasonValidatorMalformed {
t.Fatalf("expected malformed warning, got %+v", out.ModuleResults[1].Warnings)
if len(out.ModuleResults[1].Operations) != 1 || out.ModuleResults[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid {
t.Fatalf("expected invalid decision-set operation, got %+v", out.ModuleResults[1].Operations)
}
}
@@ -1024,7 +1111,7 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
if err != nil {
return contracts.ProposalResult{}, err
}
return contracts.ProposalResult{Proposals: result.Corrections, Warnings: result.Warnings}, nil
return contracts.ProposalResult{Proposals: result.Corrections, Operations: result.Operations}, nil
}
type fakeProposalStructuredClient struct {