package runner import ( "context" "errors" "os" "path/filepath" "strconv" "strings" "sync" "testing" "time" ck "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/modules" "gitea.maximumdirect.net/eric/audita/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata" ) func runnerTestConfig() config.Config { return config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 2048}, Validation: config.Validation{MaxPromptTokens: 2048}} } type fakeFactory struct { modules map[string]contracts.TranscriptModule } func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) { m, ok := f.modules[spec.InstanceName] if !ok { return nil, errors.New("module not registered") } return m, nil } type fakeModule struct { key string policy proposals.ReplacementPolicy validators []contracts.Validator proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) } type outcomeModule struct { fakeModule proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error) } func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) { failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil }} second := fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"first": failed, "second": second}}) out, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {ModuleKey: "second", InstanceName: "second"}}}) if err != nil { t.Fatal(err) } if out.FinalTranscript.Segments[0].Text != "the" || len(out.WorkUnits) != 2 || out.WorkUnits[0].State != outcomes.WorkCompleted || out.WorkUnits[1].CorrectionsApplied != 1 { t.Fatalf("recoverable failure lost later work: %+v", out) } if len(out.WorkUnits[0].Operations) != 1 || out.WorkUnits[0].Operations[0].FailureReason != outcomes.ReasonTransportFailed { t.Fatalf("missing proposal failure: %+v", out.WorkUnits[0]) } } func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) { secondSaw := make([]int, 0) partial := fakeValidator{name: "partial", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { if len(req.CandidateProposal) != 2 { t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal)) } return validators.Result{ValidatorName: "partial", Decisions: []validators.Decision{ {ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: false, Disposition: validators.DispositionUnavailable, ReasonCode: string(outcomes.ReasonTransportFailed)}, {ProposalIndex: req.CandidateProposal[1].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}, }, Operations: []outcomes.Operation{ {Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonTransportFailed}, {Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed}, }}, nil }} last := fakeValidator{name: "last", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { for _, candidate := range req.CandidateProposal { secondSaw = append(secondSaw, candidate.ProposalIndex) } return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil }} module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{ {TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x", Confidence: 1}, {TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}, }, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired}}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}}) out, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatal(err) } if out.FinalTranscript.Segments[0].Text != "teh dog" || len(secondSaw) != 1 || secondSaw[0] != 2 { t.Fatalf("unchecked candidate applied or survivor lost: text=%q final=%v", out.FinalTranscript.Segments[0].Text, secondSaw) } unit := out.WorkUnits[0] if unit.State != outcomes.WorkCompleted || unit.CorrectionsApplied != 1 || len(unit.Operations) != 3 { t.Fatalf("expected one completed unit with two operation failures: %+v", unit) } proposal := unit.Operations[0] if len(proposal.RecoveryReasons) != 2 || proposal.RecoveryReasons[0] != outcomes.RecoveryOutputRepaired || proposal.RecoveryReasons[1] != outcomes.RecoveryProposalsFiltered || len(proposal.Filtered) != 1 || proposal.Filtered[0].ProposalIndex != 0 { t.Fatalf("repair and filtering must share one recovery: %+v", proposal) } } func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) { origin := errors.New("validator setup failed") proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}} broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin }} result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []contracts.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}}) if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 { t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err) } } func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) { module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good", Confidence: 1}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}) input := RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "bad"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}} without, err := r.Run(context.Background(), input) if err != nil { t.Fatal(err) } if len(without.Candidates) != 0 { t.Fatalf("debug-off candidate capture: %+v", without.Candidates) } input.CaptureCandidates = true with, err := r.Run(context.Background(), input) if err != nil { t.Fatal(err) } if len(with.Candidates) != 1 || with.Candidates[0].ProposalIndex != 0 || len(with.ModuleResults[0].ValidatorRejected) != 1 { t.Fatalf("filtered candidate missing from debug evidence: %+v", with) } } func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { return m.proposeResult(req) } func (m fakeModule) Key() string { return m.key } func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m fakeModule) Validators() []contracts.Validator { return m.validators } func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { if m.proposeF == nil { return contracts.ProposalResult{}, nil } proposalsOut, err := m.proposeF(req) return contracts.ProposalResult{Proposals: proposalsOut}, err } type fakeValidator struct { name string validateF func(req contracts.ValidationRequest) (validators.Result, error) } func (v fakeValidator) Name() string { return v.name } func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) { _ = ctx return v.validateF(req) } type classifiedFakeValidator struct { fakeValidator class validatormetadata.ExecutionClass } func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionClass { return v.class } func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) { llm := classifiedFakeValidator{ fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { return validators.Result{ValidatorName: "llm"}, nil }}, class: validatormetadata.ExecutionClassLLMBacked, } deterministic := classifiedFakeValidator{ fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { return validators.Result{ValidatorName: "deterministic"}, nil }}, class: validatormetadata.ExecutionClassDeterministic, } ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic}) if len(ordered) != 2 { t.Fatalf("expected 2 validators, got %d", len(ordered)) } if ordered[0].Name() != "deterministic" || ordered[1].Name() != "llm" { t.Fatalf("unexpected validator order: %s, %s", ordered[0].Name(), ordered[1].Name()) } } func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) { unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { return validators.Result{ValidatorName: "plain"}, nil }} llm := classifiedFakeValidator{ fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { return validators.Result{ValidatorName: "llm"}, nil }}, class: validatormetadata.ExecutionClassLLMBacked, } ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified}) if len(ordered) != 2 { t.Fatalf("expected 2 validators, got %d", len(ordered)) } if ordered[0].Name() != "plain" || ordered[1].Name() != "llm" { t.Fatalf("unexpected validator order with unclassified validator: %s, %s", ordered[0].Name(), ordered[1].Name()) } } func TestRunnerOneModuleAppliesProposal(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "grammar": fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected corrected text, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 { t.Fatalf("expected one module with one applied change, got %+v", out.ModuleResults) } } func TestRunnerModulesRunSequentially(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil }}, "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { if req.WorkingTranscript.Segments[0].Text != "the cat" { t.Fatalf("second module did not see first module changes: %q", req.WorkingTranscript.Segments[0].Text) } return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "the dog" { t.Fatalf("expected sequential updates, got %q", out.FinalTranscript.Segments[0].Text) } } func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) { cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Speaker: "A", Start: 0, End: 1, Text: "a"}, {ID: 2, Speaker: "B", Start: 1, End: 2, Text: "b"}, {ID: 3, Speaker: "A", Start: 2, End: 3, Text: "c"}, {ID: 4, Speaker: "B", Start: 3, End: 4, Text: "d"}, }} var mu sync.Mutex seen := make(map[int][]int) updatedText := "a one two three four five" r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "grow": fakeModule{key: "grow", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { if req.Section.Index != 0 { return nil, nil } return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "a", CorrectedText: updatedText, Confidence: 1}}, nil }}, "inspect": fakeModule{key: "inspect", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { ids := make([]int, len(req.WorkingTranscript.Segments)) for i, segment := range req.WorkingTranscript.Segments { ids[i] = segment.ID } mu.Lock() seen[req.Section.Index] = ids mu.Unlock() if req.Section.Index == 0 && req.WorkingTranscript.Segments[0].Text != updatedText { return nil, errors.New("later module did not see updated section text") } if req.Section.Index == 0 { return []proposals.CorrectionProposal{{TargetSegmentID: 3, OriginalText: "c", CorrectedText: "C", Confidence: 1}}, nil } return nil, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grow", InstanceName: "grow"}, {ModuleKey: "inspect", InstanceName: "inspect"}}}) if err != nil { t.Fatal(err) } if len(out.Sections) != 2 || out.Sections[0].StartSegmentID != 1 || out.Sections[0].EndSegmentID != 2 || out.Sections[0].EstimatedTokens != 2 || len(out.WorkUnits) != 4 { t.Fatalf("initial inventory changed: sections=%+v units=%+v", out.Sections, out.WorkUnits) } if len(seen) != 2 || len(seen[0]) != 2 || seen[0][0] != 1 || seen[0][1] != 2 || len(seen[1]) != 2 || seen[1][0] != 3 || seen[1][1] != 4 { t.Fatalf("later module saw changed section membership: %+v", seen) } if len(out.FinalTranscript.Segments) != 4 || out.FinalTranscript.Segments[0].Text != updatedText || out.FinalTranscript.Segments[2].Text != "c" || out.FinalTranscript.Segments[3].ID != 4 { t.Fatalf("recombined transcript: %+v", out.FinalTranscript) } for i, segment := range out.FinalTranscript.Segments { original := transcript.Segments[i] if segment.ID != original.ID || segment.Speaker != original.Speaker || segment.Start != original.Start || segment.End != original.End { t.Fatalf("segment %d metadata changed: %+v", i, segment) } } if out.WorkUnits[0].CorrectionsApplied != 1 || out.WorkUnits[1].CorrectionsApplied != 0 { t.Fatalf("section application counts: %+v", out.WorkUnits) } } func TestRunnerInventoriesEmptyWorkAndRejectsImpossibleChunking(t *testing.T) { r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{}}) empty, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{}}) if err != nil || len(empty.Sections) != 0 || len(empty.WorkUnits) != 0 || len(empty.FinalTranscript.Segments) != 0 { t.Fatalf("empty transcript and pipeline: %+v, %v", empty, err) } input := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one"}, {ID: 2, Text: "two"}}} noModules, err := r.Run(context.Background(), RunInput{Transcript: input}) if err != nil || len(noModules.Sections) != 1 || len(noModules.WorkUnits) != 0 || len(noModules.FinalTranscript.Segments) != 2 { t.Fatalf("empty pipeline inventory: %+v, %v", noModules, err) } cfg := runnerTestConfig() target := 3 cfg.Chunking.TargetSections = &target if _, err := r.Run(context.Background(), RunInput{Config: &cfg, Transcript: input}); err == nil { t.Fatal("impossible section target was accepted") } } func TestRunnerProposalsExecutePerChunkSection(t *testing.T) { cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Text: "one two"}, {ID: 2, Text: "three four"}, {ID: 3, Text: "five six"}, }} var ( mu sync.Mutex seenBySection = make(map[int]contracts.SectionMetadata) segmentCountBySec = make(map[int]int) ) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { if req.Section == nil { t.Fatalf("expected section metadata on proposal request") } mu.Lock() seenBySection[req.Section.Index] = *req.Section segmentCountBySec[req.Section.Index] = len(req.WorkingTranscript.Segments) mu.Unlock() return nil, nil }}, }}) out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{ {ModuleKey: "m", InstanceName: "m"}, }, }) if err != nil { t.Fatalf("Run error: %v", err) } if len(out.ModuleResults) != 1 { t.Fatalf("expected one module result, got %+v", out.ModuleResults) } if len(seenBySection) != 3 { t.Fatalf("expected 3 chunked proposal calls, got %d", len(seenBySection)) } for i := 0; i < 3; i++ { section, ok := seenBySection[i] if !ok { t.Fatalf("expected section %d to be observed", i) } if section.Index != i { t.Fatalf("expected section index %d, got %+v", i, section) } if segmentCountBySec[i] != 1 { t.Fatalf("expected one segment per section call, got %d at index %d", segmentCountBySec[i], i) } } } func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) { cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Text: "alpha one"}, {ID: 2, Text: "bravo two"}, {ID: 3, Text: "charlie three"}, }} started := make(chan int, len(transcript.Segments)) releaseByID := map[int]chan struct{}{ 1: make(chan struct{}), 2: make(chan struct{}), 3: make(chan struct{}), } r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { if len(req.WorkingTranscript.Segments) != 1 { t.Fatalf("expected one segment per section, got %d", len(req.WorkingTranscript.Segments)) } seg := req.WorkingTranscript.Segments[0] started <- seg.ID <-releaseByID[seg.ID] return []proposals.CorrectionProposal{ {TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1}, }, nil }}, }}) resultCh := make(chan struct { out RunOutput err error }, 1) go func() { out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{ {ModuleKey: "m", InstanceName: "m"}, }, }) resultCh <- struct { out RunOutput err error }{out: out, err: err} }() waitForRunnerSectionIDs(t, started, map[int]struct{}{1: {}, 2: {}, 3: {}}) close(releaseByID[3]) close(releaseByID[2]) close(releaseByID[1]) result := <-resultCh out, err := result.out, result.err if err != nil { t.Fatalf("Run error: %v", err) } if len(out.ModuleResults) != 1 { t.Fatalf("expected one module result, got %+v", out.ModuleResults) } if got := len(out.ModuleResults[0].AppliedChanges); got != 3 { t.Fatalf("expected three applied changes, got %d", got) } for i, change := range out.ModuleResults[0].AppliedChanges { wantID := i + 1 if change.TargetSegmentID != wantID { t.Fatalf("expected deterministic applied-change order by section/segment, got %+v", out.ModuleResults[0].AppliedChanges) } } wantFinal := []string{"ALPHA ONE", "BRAVO TWO", "CHARLIE THREE"} for i, seg := range out.FinalTranscript.Segments { if seg.Text != wantFinal[i] { t.Fatalf("expected deterministic final transcript regardless of section completion order; got %+v", out.FinalTranscript.Segments) } } } func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) { t.Helper() deadline := time.After(350 * time.Millisecond) got := 0 for got < want { select { case <-entered: got++ case <-deadline: t.Fatalf("timed out waiting for %d entries (%s), got %d", want, label, got) } } } func waitForRunnerSectionIDs(t *testing.T, started <-chan int, want map[int]struct{}) { t.Helper() deadline := time.After(350 * time.Millisecond) seen := map[int]struct{}{} for len(seen) < len(want) { select { case id := <-started: seen[id] = struct{}{} case <-deadline: t.Fatalf("timed out waiting for section IDs %v, got %v", mapKeys(want), mapKeys(seen)) } } } func mapKeys(values map[int]struct{}) []int { keys := make([]int, 0, len(values)) for key := range values { keys = append(keys, key) } return keys } func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) { cfg := runnerTestConfig() cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Text: "alpha one"}, {ID: 2, Text: "bravo two"}, }} releaseSectionOne := make(chan struct{}) validatorStarted := make(chan struct{}, 1) validator := fakeValidator{ name: "capture", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 { select { case validatorStarted <- struct{}{}: default: } } decisions := make([]validators.Decision, 0, len(req.CandidateProposal)) for _, p := range req.CandidateProposal { decisions = append(decisions, validators.Decision{ ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok", }) } return validators.Result{ValidatorName: "capture", Decisions: decisions}, nil }, } r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{ key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { seg := req.WorkingTranscript.Segments[0] if req.Section != nil && req.Section.Index == 1 { <-releaseSectionOne } return []proposals.CorrectionProposal{ {TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1}, }, nil }, }, }}) resultCh := make(chan error, 1) go func() { _, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{ {ModuleKey: "m", InstanceName: "m"}, }, }) resultCh <- err }() select { case <-validatorStarted: case <-time.After(350 * time.Millisecond): t.Fatal("expected section-level validator work before all proposal sections complete") } close(releaseSectionOne) if err := <-resultCh; err != nil { t.Fatalf("Run error: %v", err) } } type stageAwareStructuredClient struct { startedSection chan int releaseValidation <-chan struct{} eventSink chan<- string } type captureContractStructuredClient struct { lastRequest contracts.StructuredCompletionRequest } func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx c.lastRequest = req if target, ok := out.(*validators.LLMValidationResponse); ok { *target = validators.LLMValidationResponse{ Validations: []validators.LLMValidationDecision{ {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}, }, } } return contracts.StructuredCompletionResponse{}, nil } func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { section := 0 if req.Context.SectionIndex != nil { section = *req.Context.SectionIndex } if c.startedSection != nil { select { case c.startedSection <- section: default: } } if c.eventSink != nil { c.eventSink <- "v" + strconv.Itoa(section) } if c.releaseValidation != nil { select { case <-c.releaseValidation: case <-ctx.Done(): return contracts.StructuredCompletionResponse{}, ctx.Err() } } target := out.(*validators.LLMValidationResponse) *target = validators.LLMValidationResponse{ Validations: []validators.LLMValidationDecision{ {CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"}, }, } return contracts.StructuredCompletionResponse{}, nil } func TestRunnerSkippedRecorded(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if len(out.ModuleResults[0].SkippedChanges) != 1 { t.Fatalf("expected one skipped change, got %+v", out.ModuleResults[0].SkippedChanges) } } func TestRunnerFailureReturnsPartialProgress(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil }}, "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return nil, errors.New("boom") }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}}) if err == nil { t.Fatal("expected error") } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected partial transcript progress preserved, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed { t.Fatalf("expected second module failed in results, got %+v", out.ModuleResults) } } func TestRunnerDoesNotMutateInputTranscript(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}} before := transcript.Segments[0].Text r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil }}, }}) _, err := r.Run(context.Background(), RunInput{Config: ptrConfig(runnerTestConfig()), Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if transcript.Segments[0].Text != before { t.Fatalf("expected input transcript unchanged, got %q", transcript.Segments[0].Text) } } func TestRunnerRepeatedModuleInstanceNames(t *testing.T) { specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary"}) if err != nil { t.Fatalf("ResolveModuleRunSpecs error: %v", err) } if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" { t.Fatalf("unexpected instance names: %+v", specs) } } func TestRunnerValidatorApprovedProposalApplied(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}} allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { decisions := make([]validators.Decision, len(req.CandidateProposal)) for i, p := range req.CandidateProposal { decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"} } return validators.Result{ValidatorName: "allow", Decisions: decisions}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{ key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allowAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil }, }, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected proposal applied, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults[0].ValidatorDecisions) != 1 { t.Fatalf("expected validator decisions recorded") } if len(out.ValidatorTimings) != 1 || out.ValidatorTimings[0].Calls != 1 || out.ValidatorTimings[0].ValidatorName != "allow" || out.RunWallTimeMS < 0 { t.Fatalf("runner timings missing: %+v", out.ValidatorTimings) } } func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}} rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { decisions := make([]validators.Decision, len(req.CandidateProposal)) for i, p := range req.CandidateProposal { decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"} } return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "teh cat" { t.Fatalf("expected rejected proposal not applied, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults[0].ValidatorRejected) != 1 { t.Fatalf("expected validator rejection recorded, got %+v", out.ModuleResults[0].ValidatorRejected) } } func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}} first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { if len(req.CandidateProposal) != 2 { t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal)) } return validators.Result{ ValidatorName: "first", Decisions: []validators.Decision{ {ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}, {ProposalIndex: req.CandidateProposal[1].ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "reject"}, }, }, nil }} second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { if len(req.CandidateProposal) != 1 { t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal)) } return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE", Confidence: 1}, {TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO", Confidence: 1}, }, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "ONE two" { t.Fatalf("expected only survivor applied, got %q", out.FinalTranscript.Segments[0].Text) } } func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}} good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil }} badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { // Missing one decision triggers cardinality error. return validators.Result{ValidatorName: "bad", Decisions: nil}, nil }} bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}}) if err == nil { t.Fatal("expected cardinality error") } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected partial progress preserved, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed { t.Fatalf("expected second module failed") } } func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}} allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}}) if err != nil { t.Fatalf("Run error: %v", err) } if len(out.ModuleResults[0].SkippedChanges) != 1 { t.Fatalf("expected application skip recorded") } if len(out.ModuleResults[0].ValidatorRejected) != 0 { t.Fatalf("expected no validator rejection") } } func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": outcomeModule{ fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{ Proposals: []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "a", CorrectedText: "", Confidence: 1}, {TargetSegmentID: 1, OriginalText: "b", CorrectedText: "", Confidence: 1}, }, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}, }, nil }, }, }}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, CaptureCandidates: true}) if err != nil { t.Fatal(err) } if got := out.FinalTranscript.Segments[0].Text; got != " b" { t.Fatalf("expected first deletion only, got %q", got) } if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].SkippedChanges) != 1 || out.ModuleResults[0].SkippedChanges[0].SkipReason != proposals.SkipReasonEmptyResultingText { t.Fatalf("expected skipped debug evidence: %+v", out.ModuleResults) } if len(out.ModuleResults[0].ValidatorRejected) != 0 || len(out.WorkUnits) != 1 || out.WorkUnits[0].CorrectionsApplied != 1 { t.Fatalf("unexpected validation or application outcome: %+v", out) } foundProposal := false for _, operation := range out.WorkUnits[0].Operations { if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { foundProposal = true } if operation.FailureReason != "" || len(operation.RecoveryReasons) != 0 || len(operation.Filtered) != 0 { t.Fatalf("apply-time conflict must not count as model error or filtering recovery: %+v", operation) } } if !foundProposal { t.Fatalf("expected successful proposal operation: %+v", out.WorkUnits[0].Operations) } } func ptrConfig(c config.Config) *config.Config { return &c } type fakeStructuredClient struct { responses []validators.LLMValidationResponse err error calls int } func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx _ = req f.calls++ if f.err != nil { return contracts.StructuredCompletionResponse{}, f.err } if len(f.responses) == 0 { return contracts.StructuredCompletionResponse{}, errors.New("unexpected call") } resp := f.responses[0] f.responses = f.responses[1:] target := out.(*validators.LLMValidationResponse) *target = resp return contracts.StructuredCompletionResponse{}, nil } func TestRunnerLLMValidatorApprovalApplied(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}}} llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) if err != nil { t.Fatalf("NewLLMBackedValidator: %v", err) } r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil }}, }}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "There were Jesters at the temple." { t.Fatalf("expected applied LLM-approved proposal, got %q", out.FinalTranscript.Segments[0].Text) } } func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "reject"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil }}, }}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "There were gestures at the temple." { t.Fatalf("expected rejected proposal not applied") } if len(out.ModuleResults[0].ValidatorRejected) != 1 { t.Fatalf("expected validator rejection record") } } func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil }}, "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil }}, }}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}, ValidationLLMClient: client, }) if err != nil { t.Fatalf("expected malformed validator response to downgrade, got %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected partial progress retained") } if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 { t.Fatalf("expected second module rejection, got %+v", out.ModuleResults) } if len(out.ModuleResults[1].Operations) != 1 || out.ModuleResults[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid { t.Fatalf("expected invalid decision-set operation, got %+v", out.ModuleResults[1].Operations) } } func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ {CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}, {CorrectionIndex: 3, Approved: true, Confidence: .9, Reason: "ok"}, {CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"}, }}}} first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) var laterIndices []int later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { decisions := make([]validators.Decision, 0, len(req.CandidateProposal)) for _, candidate := range req.CandidateProposal { laterIndices = append(laterIndices, candidate.ProposalIndex) approved := candidate.ProposalIndex == 0 disposition := validators.DispositionRejected if approved { disposition = validators.DispositionApproved } decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: approved, Disposition: disposition, ReasonCode: "later_review"}) } return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil }} module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}, {TargetSegmentID: 1, OriginalText: "bad", CorrectedText: "good", Confidence: 1}, }, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}}) cfg := runnerTestConfig() out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 { t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err) } unit := out.WorkUnits[0] if unit.CorrectionsApplied != 1 || len(unit.Operations) != 2 || !unit.Operations[0].Succeeded || unit.Operations[1].FailureReason != outcomes.ReasonValidationDecisionsInvalid || len(unit.Operations[1].ProposalIndices) != 1 || unit.Operations[1].ProposalIndices[0] != 1 || len(unit.Operations[1].RecoveryReasons) != 0 { t.Fatalf("operation accounting: %+v", unit) } moduleResult := out.ModuleResults[0] if len(moduleResult.ValidatorDecisions) != 5 || len(moduleResult.ValidatorRejected) != 2 || moduleResult.ValidatorDecisions[2].Disposition != validators.DispositionUnavailable || moduleResult.ValidatorRejected[0].Disposition != validators.DispositionUnavailable || moduleResult.ValidatorRejected[1].Disposition != validators.DispositionRejected { t.Fatalf("validator dispositions: %+v", moduleResult) } } func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil }}, }}) cfg := runnerTestConfig() _, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) if err != nil { t.Fatalf("expected missing decision downgrade, got %v", err) } } func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ {CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}, {CorrectionIndex: 1, Approved: false, Confidence: 0.9, Reason: "dup"}, }}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil }}, }}) cfg := runnerTestConfig() _, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) if err != nil { t.Fatalf("expected duplicate decision downgrade, got %v", err) } } type lenEstimator struct{} func (lenEstimator) EstimateTokens(text string) int { return len(text) } func TestRunnerLLMValidatorBatching(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{ {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}, {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}, }} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator.SetTokenEstimator(lenEstimator{}) cfg := runnerTestConfig() cfg.Validation.MaxPromptTokens = 260 r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40), Confidence: 1}, {TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40), Confidence: 1}, }, nil }}, }}) out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, }) if err != nil { t.Fatalf("unexpected run error: %v", err) } if client.calls < 2 { t.Fatalf("expected multiple llm calls for batching, got %d", client.calls) } if len(out.ModuleResults[0].AppliedChanges) != 2 { t.Fatalf("expected both approved proposals applied") } } func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { primarySecret := "runner-primary-secret" validationSecret := "runner-validation-secret" client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) t.Setenv("RUNNER_PRIMARY_KEY", primarySecret) t.Setenv("RUNNER_VALIDATION_KEY", validationSecret) loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: RUNNER_PRIMARY_KEY\n validation:\n endpoint: https://validation.invalid/v1\n credential: RUNNER_VALIDATION_KEY\nllm:\n proposal_profile: base\n")), "") if loadErr != nil { t.Fatal(loadErr) } cfg := loaded.Config diagDir := t.TempDir() r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil }}, }}) out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client, ValidationDiagnosticsDir: diagDir, }) if err != nil { t.Fatalf("Run error: %v", err) } if len(out.ModuleResults[0].ValidatorDecisions) == 0 || out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath == "" { t.Fatalf("expected diagnostic artifact path on decision") } matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*.json")) if globErr != nil { t.Fatalf("glob diagnostics: %v", globErr) } if len(matches) == 0 { t.Fatalf("expected diagnostics JSON artifacts under %s", filepath.Join(diagDir, "m")) } for _, path := range matches { raw, readErr := os.ReadFile(path) if readErr != nil { t.Fatalf("read diagnostic %q: %v", path, readErr) } if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) { t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw)) } } raw, readErr := os.ReadFile(out.ModuleResults[0].ValidatorDecisions[0].DiagnosticArtifactPath) if readErr != nil { t.Fatalf("read decision diagnostic: %v", readErr) } if !strings.Contains(string(raw), "[REDACTED]") { t.Fatalf("expected redaction marker in diagnostics") } } type proposalGenerationModule struct { key string policy proposals.ReplacementPolicy } func (m proposalGenerationModule) Key() string { return m.key } func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } func (m proposalGenerationModule) Validators() []contracts.Validator { return nil } func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ ModuleKey: req.RunSpec.ModuleKey, ModuleInstance: req.RunSpec.InstanceName, ReplacementPolicy: req.RunSpec.ReplacementPolicy, WorkingTranscript: req.WorkingTranscript, Config: req.Config, Glossary: req.Glossary, PromptID: "modules.grammar.proposal", PromptVersion: "v1", Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, LLMClient: req.LLMClient, DiagnosticsDir: req.DiagnosticsDir, }) if err != nil { return contracts.ProposalResult{}, err } return contracts.ProposalResult{Proposals: result.Corrections, Operations: result.Operations}, nil } type fakeProposalStructuredClient struct { responses []proposal_generation.StructuredCorrectionSet } func (f *fakeProposalStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx _ = req if len(f.responses) == 0 { return contracts.StructuredCompletionResponse{}, errors.New("unexpected call") } target, ok := out.(*proposal_generation.StructuredCorrectionSet) if !ok { return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type") } *target = f.responses[0] f.responses = f.responses[1:] return contracts.StructuredCompletionResponse{}, nil } func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) { client := &fakeProposalStructuredClient{ responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ {SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}, }, }, }, } cfg := runnerTestConfig() diagDir := t.TempDir() r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": proposalGenerationModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique}, }}) out, err := r.Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ProposalLLMClient: client, ProposalDiagnosticsDir: diagDir, }) if err != nil { t.Fatalf("unexpected run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "the cat" { t.Fatalf("expected proposal-generated correction to apply, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 { t.Fatalf("expected one applied change, got %+v", out.ModuleResults) } matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*proposal-generation*response-payload.json")) if globErr != nil { t.Fatalf("glob diagnostics: %v", globErr) } if len(matches) == 0 { t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m")) } } func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) { client := &fakeProposalStructuredClient{ responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ {SegmentReference: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5}, }, }, }, } cfg := runnerTestConfig() cfg.Thresholds.Grammar = 0.9 factory := modules.NewFactory(modules.Dependencies{}) out, err := New(factory).Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}}, Glossary: &schema.Glossary{}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, ProposalLLMClient: client, }) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "hello ,world" { t.Fatalf("expected no changes due to confidence threshold, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].ValidatorRejected) == 0 { t.Fatalf("expected validator rejection, got %+v", out.ModuleResults) } if out.ModuleResults[0].ValidatorRejected[0].ReasonCode != validators.ReasonLowConfidence { t.Fatalf("expected low confidence reason, got %+v", out.ModuleResults[0].ValidatorRejected[0]) } } func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) { client := &fakeProposalStructuredClient{ responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ {SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5}, }, }, }, } cfg := runnerTestConfig() cfg.Thresholds.Glossary = 0.9 factory := modules.NewFactory(modules.Dependencies{}) out, err := New(factory).Run(context.Background(), RunInput{ Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}}, Glossary: &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary"}}, ProposalLLMClient: client, }) if err != nil { t.Fatalf("Run error: %v", err) } if out.FinalTranscript.Segments[0].Text != "There were gestures" { t.Fatalf("expected no changes due to glossary confidence threshold, got %q", out.FinalTranscript.Segments[0].Text) } if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].ValidatorRejected) == 0 { t.Fatalf("expected validator rejection, got %+v", out.ModuleResults) } if out.ModuleResults[0].ValidatorRejected[0].ReasonCode != validators.ReasonLowConfidence { t.Fatalf("expected low confidence reason, got %+v", out.ModuleResults[0].ValidatorRejected[0]) } } func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) { origin := errors.New("original fatal failure") candidate := proposals.EnrichedCorrectionProposal{ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}} for _, decisions := range [][]validators.Decision{ {{ProposalIndex: 8, Approved: true}}, {{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}}, } { v := fakeValidator{name: "broken", validateF: func(contracts.ValidationRequest) (validators.Result, error) { return validators.Result{Decisions: decisions}, origin }} result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []contracts.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}) if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 { t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err) } } }