package homophones import ( "context" "errors" "os" "path/filepath" "strings" "testing" "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/validators" ) type fakeLLMClient struct { responses []proposal_generation.StructuredCorrectionSet err error calls []contracts.StructuredCompletionRequest } func (f *fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx f.calls = append(f.calls, req) if f.err != nil { return contracts.StructuredCompletionResponse{}, f.err } target, ok := out.(*proposal_generation.StructuredCorrectionSet) if !ok { return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type") } if len(f.responses) == 0 { return contracts.StructuredCompletionResponse{}, errors.New("unexpected call") } *target = f.responses[0] f.responses = f.responses[1:] return contracts.StructuredCompletionResponse{}, nil } type countingScheduler struct { runs int } func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error { s.runs++ return fn(ctx) } func tinyTranscript() *schema.Transcript { return &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Speaker: "A", Start: 0, End: 1, Text: "There were gestures in the hall.", Categories: []string{"session"}}, }} } func tinyGlossary() *schema.Glossary { return &schema.Glossary{Entries: []schema.GlossaryEntry{ {Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"}, }} } func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testing.T) { msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "") if err != nil { t.Fatalf("BuildProposalMessages error: %v", err) } if len(msgs) != 2 { t.Fatalf("expected 2 messages, got %d", len(msgs)) } combined := msgs[0].Content + "\n" + msgs[1].Content for _, want := range []string{ "Protected glossary/context:", "Transcript section:", `"section_index": 0`, `"Aliases":`, `"Category": "faction"`, `"Summary": "Guild members"`, `"Plural": "Jesters"`, "homophone correction assistant", "mistranscriptions of spoken English", "Do not add or remove punctuation", "Do not return speaker, start, or end fields", `"id": 1`, } { if !strings.Contains(combined, want) { t.Fatalf("expected prompt to contain %q", want) } } for _, forbidden := range []string{ "summarize the transcript", "style rewrite", "grammar cleanup", "punctuation cleanup", } { if strings.Contains(strings.ToLower(combined), forbidden) { t.Fatalf("prompt should not invite %q", forbidden) } } } func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) { msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Tabletop session with fantasy names.") if err != nil { t.Fatalf("BuildProposalMessages error: %v", err) } combined := msgs[0].Content + "\n" + msgs[1].Content if !strings.Contains(combined, "Transcript description (background context only):") { t.Fatalf("expected transcript description section in prompt") } if !strings.Contains(combined, "must not override the transcript content") { t.Fatalf("expected no-override guidance in prompt") } } func TestBuildProposalMessagesOmitsTranscriptDescriptionSectionWhenEmpty(t *testing.T) { msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, " ") if err != nil { t.Fatalf("BuildProposalMessages error: %v", err) } combined := msgs[0].Content + "\n" + msgs[1].Content if strings.Contains(combined, "Transcript description (background context only):") { t.Fatalf("did not expect empty transcript description section in prompt") } } func TestHomophonesModuleReplacementPolicy(t *testing.T) { m, err := New() if err != nil { t.Fatalf("New error: %v", err) } if m.ReplacementPolicy() != proposals.ReplacementPolicyRequireUnique { t.Fatalf("unexpected replacement policy: %q", m.ReplacementPolicy()) } } func TestHomophonesModuleValidatorChain(t *testing.T) { m, err := New() if err != nil { t.Fatalf("New error: %v", err) } got := make([]string, 0, len(m.Validators())) for _, v := range m.Validators() { got = append(got, v.Name()) } want := []string{ "no_effect", "original_text_presence", "confidence_threshold", "protected_glossary_terms", "non_empty_correction", "spoken_form_plausibility_review", "meaning_reversal_review", } if strings.Join(got, ",") != strings.Join(want, ",") { t.Fatalf("unexpected validator chain\n got: %v\nwant: %v", got, want) } } func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) { secret := "homophones-secret" client := &fakeLLMClient{ responses: []proposal_generation.StructuredCorrectionSet{ { Corrections: []proposal_generation.StructuredCorrectionProposal{ {TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94}, }, }, }, } scheduler := &countingScheduler{} cfg := config.Default() cfg.PrimaryLLM.APIKey = secret m, err := New() if err != nil { t.Fatalf("New error: %v", err) } diagDir := t.TempDir() out, err := m.Propose(context.Background(), contracts.ProposalRequest{ ExecutionContext: contracts.ExecutionContext{ Config: &cfg, WorkingTranscript: tinyTranscript(), Glossary: tinyGlossary(), DiagnosticsDir: diagDir, }, RunSpec: contracts.ModuleRunSpec{ ModuleKey: "homophones", InstanceName: "homophones", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, }, LLMClient: client, LLMScheduler: scheduler, }) if err != nil { t.Fatalf("Propose error: %v", err) } if scheduler.runs != 1 { t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs) } if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" { t.Fatalf("expected one homophones:proposal call, got %+v", client.calls) } if len(out) != 1 || out[0].CorrectedText != "Jesters" { t.Fatalf("unexpected proposals: %+v", out) } diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json")) if globErr != nil { t.Fatalf("glob error: %v", globErr) } if len(diagFiles) == 0 { t.Fatalf("expected diagnostics response payload under %s", filepath.Join(diagDir, "homophones")) } for _, f := range diagFiles { raw, readErr := os.ReadFile(f) if readErr != nil { t.Fatalf("read diag %q: %v", f, readErr) } if strings.Contains(string(raw), secret) { t.Fatalf("secret leaked in diagnostics: %s", string(raw)) } } } func TestHomophonesConfidenceThresholdIsUsed(t *testing.T) { module, err := New() if err != nil { t.Fatalf("New error: %v", err) } var thresholdValidator contracts.Validator for _, v := range module.Validators() { if v.Name() == "confidence_threshold" { thresholdValidator = v break } } if thresholdValidator == nil { t.Fatal("expected confidence_threshold validator") } cfg := config.Default() cfg.Thresholds.Homophones = 0.95 result, err := thresholdValidator.Validate(context.Background(), validators.Request{ Config: &cfg, ModuleKey: "homophones", CandidateProposal: []proposals.EnrichedCorrectionProposal{ { CorrectionProposal: proposals.CorrectionProposal{ TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94, }, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"}, }, }, }) if err != nil { t.Fatalf("validate error: %v", err) } if len(result.Decisions) != 1 { t.Fatalf("expected one decision, got %+v", result.Decisions) } if result.Decisions[0].Approved { t.Fatalf("expected rejection under homophones threshold, got %+v", result.Decisions[0]) } if result.Decisions[0].ReasonCode != validators.ReasonLowConfidence { t.Fatalf("expected low confidence reason, got %+v", result.Decisions[0]) } } func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testing.T) { module, err := New() if err != nil { t.Fatalf("New error: %v", err) } var protectedValidator contracts.Validator for _, v := range module.Validators() { if v.Name() == "protected_glossary_terms" { protectedValidator = v break } } if protectedValidator == nil { t.Fatal("expected protected_glossary_terms validator") } working := &schema.Transcript{Segments: []schema.Segment{ {ID: 1, Speaker: "A", Start: 0, End: 1, Text: "Jesters are here."}, }} glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{ {Name: "Jesters", Category: "faction", Summary: "Faction"}, }} result, err := protectedValidator.Validate(context.Background(), validators.Request{ WorkingTranscript: working, Glossary: glossary, ModuleKey: "homophones", CandidateProposal: []proposals.EnrichedCorrectionProposal{ { CorrectionProposal: proposals.CorrectionProposal{ TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "gestures", Confidence: 0.99, }, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"}, }, }, }) if err != nil { t.Fatalf("validate error: %v", err) } if len(result.Decisions) != 1 { t.Fatalf("expected one decision, got %+v", result.Decisions) } if result.Decisions[0].Approved { t.Fatalf("expected protected-term rejection, got %+v", result.Decisions[0]) } if result.Decisions[0].ReasonCode != validators.ReasonProtectedGlossaryTerm { t.Fatalf("expected protected glossary-term reason, got %+v", result.Decisions[0]) } }