310 lines
16 KiB
Go
310 lines
16 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
|
)
|
|
|
|
type fakeFactory struct {
|
|
modules map[string]contracts.TranscriptModule
|
|
}
|
|
|
|
func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
|
|
m, ok := f.modules[spec.InstanceName]
|
|
if !ok {
|
|
return nil, errors.New("module not registered")
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
type fakeModule struct {
|
|
key string
|
|
policy proposals.ReplacementPolicy
|
|
validators []contracts.Validator
|
|
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
|
|
}
|
|
|
|
func (m fakeModule) Key() string { return m.key }
|
|
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
|
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
|
|
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
if m.proposeF == nil {
|
|
return nil, nil
|
|
}
|
|
return m.proposeF(req)
|
|
}
|
|
|
|
type fakeValidator struct {
|
|
name string
|
|
validateF func(req contracts.ValidationRequest) (validators.Result, error)
|
|
}
|
|
|
|
func (v fakeValidator) Name() string { return v.name }
|
|
func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) {
|
|
_ = ctx
|
|
return v.validateF(req)
|
|
}
|
|
|
|
func TestRunnerOneModuleAppliesProposal(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"grammar": fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}, nil
|
|
}},
|
|
}})
|
|
|
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}})
|
|
if err != nil {
|
|
t.Fatalf("Run error: %v", err)
|
|
}
|
|
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
|
t.Fatalf("expected corrected text, got %q", out.FinalTranscript.Segments[0].Text)
|
|
}
|
|
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 {
|
|
t.Fatalf("expected one module with one applied change, got %+v", out.ModuleResults)
|
|
}
|
|
}
|
|
|
|
func TestRunnerModulesRunSequentially(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
|
}},
|
|
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
if req.WorkingTranscript.Segments[0].Text != "the cat" {
|
|
t.Fatalf("second module did not see first module changes: %q", req.WorkingTranscript.Segments[0].Text)
|
|
}
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
|
|
}},
|
|
}})
|
|
|
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}})
|
|
if err != nil {
|
|
t.Fatalf("Run error: %v", err)
|
|
}
|
|
if out.FinalTranscript.Segments[0].Text != "the dog" {
|
|
t.Fatalf("expected sequential updates, got %q", out.FinalTranscript.Segments[0].Text)
|
|
}
|
|
}
|
|
|
|
func TestRunnerSkippedRecorded(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}}, nil
|
|
}},
|
|
}})
|
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
|
if err != nil {
|
|
t.Fatalf("Run error: %v", err)
|
|
}
|
|
if len(out.ModuleResults[0].SkippedChanges) != 1 {
|
|
t.Fatalf("expected one skipped change, got %+v", out.ModuleResults[0].SkippedChanges)
|
|
}
|
|
}
|
|
|
|
func TestRunnerFailureReturnsPartialProgress(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
|
}},
|
|
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return nil, errors.New("boom")
|
|
}},
|
|
}})
|
|
|
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}})
|
|
if err == nil {
|
|
t.Fatal("expected error")
|
|
}
|
|
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
|
t.Fatalf("expected partial transcript progress preserved, got %q", out.FinalTranscript.Segments[0].Text)
|
|
}
|
|
if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed {
|
|
t.Fatalf("expected second module failed in results, got %+v", out.ModuleResults)
|
|
}
|
|
}
|
|
|
|
func TestRunnerDoesNotMutateInputTranscript(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}}
|
|
before := transcript.Segments[0].Text
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
|
}},
|
|
}})
|
|
|
|
_, err := r.Run(context.Background(), RunInput{Config: ptrConfig(config.Default()), Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
|
if err != nil {
|
|
t.Fatalf("Run error: %v", err)
|
|
}
|
|
if transcript.Segments[0].Text != before {
|
|
t.Fatalf("expected input transcript unchanged, got %q", transcript.Segments[0].Text)
|
|
}
|
|
}
|
|
|
|
func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
|
|
specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary"})
|
|
if err != nil {
|
|
t.Fatalf("ResolveModuleRunSpecs error: %v", err)
|
|
}
|
|
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" {
|
|
t.Fatalf("unexpected instance names: %+v", specs)
|
|
}
|
|
}
|
|
|
|
func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
|
allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
|
decisions := make([]validators.Decision, len(req.CandidateProposal))
|
|
for i, p := range req.CandidateProposal {
|
|
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"}
|
|
}
|
|
return validators.Result{ValidatorName: "allow", Decisions: decisions}, nil
|
|
}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"m": fakeModule{
|
|
key: "m",
|
|
policy: proposals.ReplacementPolicyRequireUnique,
|
|
validators: []contracts.Validator{allowAll},
|
|
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
|
},
|
|
},
|
|
}})
|
|
|
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
|
if err != nil {
|
|
t.Fatalf("Run error: %v", err)
|
|
}
|
|
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
|
t.Fatalf("expected proposal applied, got %q", out.FinalTranscript.Segments[0].Text)
|
|
}
|
|
if len(out.ModuleResults[0].ValidatorDecisions) != 1 {
|
|
t.Fatalf("expected validator decisions recorded")
|
|
}
|
|
}
|
|
|
|
func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
|
rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
|
decisions := make([]validators.Decision, len(req.CandidateProposal))
|
|
for i, p := range req.CandidateProposal {
|
|
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"}
|
|
}
|
|
return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil
|
|
}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
|
}},
|
|
}})
|
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
|
if err != nil {
|
|
t.Fatalf("Run error: %v", err)
|
|
}
|
|
if out.FinalTranscript.Segments[0].Text != "teh cat" {
|
|
t.Fatalf("expected rejected proposal not applied, got %q", out.FinalTranscript.Segments[0].Text)
|
|
}
|
|
if len(out.ModuleResults[0].ValidatorRejected) != 1 {
|
|
t.Fatalf("expected validator rejection recorded, got %+v", out.ModuleResults[0].ValidatorRejected)
|
|
}
|
|
}
|
|
|
|
func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}}
|
|
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
|
if len(req.CandidateProposal) != 2 {
|
|
t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal))
|
|
}
|
|
return validators.Result{
|
|
ValidatorName: "first",
|
|
Decisions: []validators.Decision{
|
|
{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"},
|
|
{ProposalIndex: req.CandidateProposal[1].ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "reject"},
|
|
},
|
|
}, nil
|
|
}}
|
|
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
|
if len(req.CandidateProposal) != 1 {
|
|
t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal))
|
|
}
|
|
return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
|
|
}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{
|
|
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE", Confidence: 1},
|
|
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO", Confidence: 1},
|
|
}, nil
|
|
}},
|
|
}})
|
|
|
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
|
if err != nil {
|
|
t.Fatalf("Run error: %v", err)
|
|
}
|
|
if out.FinalTranscript.Segments[0].Text != "ONE two" {
|
|
t.Fatalf("expected only survivor applied, got %q", out.FinalTranscript.Segments[0].Text)
|
|
}
|
|
}
|
|
|
|
func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
|
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
|
}}
|
|
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
|
// Missing one decision triggers cardinality error.
|
|
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
|
|
}}
|
|
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
|
|
}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
|
|
|
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}}})
|
|
if err == nil {
|
|
t.Fatal("expected cardinality error")
|
|
}
|
|
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
|
t.Fatalf("expected partial progress preserved, got %q", out.FinalTranscript.Segments[0].Text)
|
|
}
|
|
if len(out.ModuleResults) != 2 || out.ModuleResults[1].Status != ModuleStatusFailed {
|
|
t.Fatalf("expected second module failed")
|
|
}
|
|
}
|
|
|
|
func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
|
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}}
|
|
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
|
return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
|
|
}}
|
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
|
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
|
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}}, nil
|
|
}},
|
|
}})
|
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
|
if err != nil {
|
|
t.Fatalf("Run error: %v", err)
|
|
}
|
|
if len(out.ModuleResults[0].SkippedChanges) != 1 {
|
|
t.Fatalf("expected application skip recorded")
|
|
}
|
|
if len(out.ModuleResults[0].ValidatorRejected) != 0 {
|
|
t.Fatalf("expected no validator rejection")
|
|
}
|
|
}
|
|
|
|
func ptrConfig(c config.Config) *config.Config { return &c }
|