324 lines
9.9 KiB
Go
324 lines
9.9 KiB
Go
package homophones
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
|
)
|
|
|
|
type fakeLLMClient struct {
|
|
responses []proposal_generation.StructuredCorrectionSet
|
|
err error
|
|
calls []contracts.StructuredCompletionRequest
|
|
}
|
|
|
|
func (f *fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
_ = ctx
|
|
f.calls = append(f.calls, req)
|
|
if f.err != nil {
|
|
return contracts.StructuredCompletionResponse{}, f.err
|
|
}
|
|
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
|
|
if !ok {
|
|
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
|
|
}
|
|
if len(f.responses) == 0 {
|
|
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
|
|
}
|
|
*target = f.responses[0]
|
|
f.responses = f.responses[1:]
|
|
return contracts.StructuredCompletionResponse{}, nil
|
|
}
|
|
|
|
type countingScheduler struct {
|
|
runs int
|
|
}
|
|
|
|
func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
|
s.runs++
|
|
return fn(ctx)
|
|
}
|
|
|
|
func tinyTranscript() *schema.Transcript {
|
|
return &schema.Transcript{Segments: []schema.Segment{
|
|
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "There were gestures in the hall.", Categories: []string{"session"}},
|
|
}}
|
|
}
|
|
|
|
func tinyGlossary() *schema.Glossary {
|
|
return &schema.Glossary{Entries: []schema.GlossaryEntry{
|
|
{Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"},
|
|
}}
|
|
}
|
|
|
|
func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testing.T) {
|
|
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
|
|
if err != nil {
|
|
t.Fatalf("BuildProposalMessages error: %v", err)
|
|
}
|
|
if len(msgs) != 2 {
|
|
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
|
}
|
|
combined := msgs[0].Content + "\n" + msgs[1].Content
|
|
for _, want := range []string{
|
|
"Protected glossary/context:",
|
|
"Transcript section:",
|
|
`"section_index": 0`,
|
|
`"Aliases":`,
|
|
`"Category": "faction"`,
|
|
`"Summary": "Guild members"`,
|
|
`"Plural": "Jesters"`,
|
|
"homophone correction assistant",
|
|
"mistranscriptions of spoken English",
|
|
"Do not add or remove punctuation",
|
|
"Do not return speaker, start, or end fields",
|
|
`"id": 1`,
|
|
} {
|
|
if !strings.Contains(combined, want) {
|
|
t.Fatalf("expected prompt to contain %q", want)
|
|
}
|
|
}
|
|
for _, forbidden := range []string{
|
|
"summarize the transcript",
|
|
"style rewrite",
|
|
"grammar cleanup",
|
|
"punctuation cleanup",
|
|
} {
|
|
if strings.Contains(strings.ToLower(combined), forbidden) {
|
|
t.Fatalf("prompt should not invite %q", forbidden)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
|
|
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Tabletop session with fantasy names.")
|
|
if err != nil {
|
|
t.Fatalf("BuildProposalMessages error: %v", err)
|
|
}
|
|
combined := msgs[0].Content + "\n" + msgs[1].Content
|
|
if !strings.Contains(combined, "Transcript description (background context only):") {
|
|
t.Fatalf("expected transcript description section in prompt")
|
|
}
|
|
if !strings.Contains(combined, "must not override the transcript content") {
|
|
t.Fatalf("expected no-override guidance in prompt")
|
|
}
|
|
}
|
|
|
|
func TestBuildProposalMessagesOmitsTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
|
|
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, " ")
|
|
if err != nil {
|
|
t.Fatalf("BuildProposalMessages error: %v", err)
|
|
}
|
|
combined := msgs[0].Content + "\n" + msgs[1].Content
|
|
if strings.Contains(combined, "Transcript description (background context only):") {
|
|
t.Fatalf("did not expect empty transcript description section in prompt")
|
|
}
|
|
}
|
|
|
|
func TestHomophonesModuleReplacementPolicy(t *testing.T) {
|
|
m, err := New()
|
|
if err != nil {
|
|
t.Fatalf("New error: %v", err)
|
|
}
|
|
if m.ReplacementPolicy() != proposals.ReplacementPolicyRequireUnique {
|
|
t.Fatalf("unexpected replacement policy: %q", m.ReplacementPolicy())
|
|
}
|
|
}
|
|
|
|
func TestHomophonesModuleValidatorChain(t *testing.T) {
|
|
m, err := New()
|
|
if err != nil {
|
|
t.Fatalf("New error: %v", err)
|
|
}
|
|
got := make([]string, 0, len(m.Validators()))
|
|
for _, v := range m.Validators() {
|
|
got = append(got, v.Name())
|
|
}
|
|
want := []string{
|
|
"proposal_shape",
|
|
"no_effect",
|
|
"original_text_presence",
|
|
"confidence_threshold",
|
|
"protected_terms",
|
|
"non_empty_corrected_text",
|
|
"spoken_form_plausibility",
|
|
"meaning_reversal_review",
|
|
}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Fatalf("unexpected validator chain\n got: %v\nwant: %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
|
|
secret := "homophones-secret"
|
|
client := &fakeLLMClient{
|
|
responses: []proposal_generation.StructuredCorrectionSet{
|
|
{
|
|
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
|
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
scheduler := &countingScheduler{}
|
|
cfg := config.Default()
|
|
cfg.PrimaryLLM.APIKey = secret
|
|
m, err := New()
|
|
if err != nil {
|
|
t.Fatalf("New error: %v", err)
|
|
}
|
|
diagDir := t.TempDir()
|
|
out, err := m.Propose(context.Background(), contracts.ProposalRequest{
|
|
ExecutionContext: contracts.ExecutionContext{
|
|
Config: &cfg,
|
|
WorkingTranscript: tinyTranscript(),
|
|
Glossary: tinyGlossary(),
|
|
DiagnosticsDir: diagDir,
|
|
},
|
|
RunSpec: contracts.ModuleRunSpec{
|
|
ModuleKey: "homophones",
|
|
InstanceName: "homophones",
|
|
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
|
},
|
|
LLMClient: client,
|
|
LLMScheduler: scheduler,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Propose error: %v", err)
|
|
}
|
|
if scheduler.runs != 1 {
|
|
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
|
}
|
|
if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" {
|
|
t.Fatalf("expected one homophones:proposal call, got %+v", client.calls)
|
|
}
|
|
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
|
|
t.Fatalf("unexpected proposals: %+v", out)
|
|
}
|
|
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json"))
|
|
if globErr != nil {
|
|
t.Fatalf("glob error: %v", globErr)
|
|
}
|
|
if len(diagFiles) == 0 {
|
|
t.Fatalf("expected diagnostics response payload under %s", filepath.Join(diagDir, "homophones"))
|
|
}
|
|
for _, f := range diagFiles {
|
|
raw, readErr := os.ReadFile(f)
|
|
if readErr != nil {
|
|
t.Fatalf("read diag %q: %v", f, readErr)
|
|
}
|
|
if strings.Contains(string(raw), secret) {
|
|
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestHomophonesConfidenceThresholdIsUsed(t *testing.T) {
|
|
module, err := New()
|
|
if err != nil {
|
|
t.Fatalf("New error: %v", err)
|
|
}
|
|
var thresholdValidator contracts.Validator
|
|
for _, v := range module.Validators() {
|
|
if v.Name() == "confidence_threshold" {
|
|
thresholdValidator = v
|
|
break
|
|
}
|
|
}
|
|
if thresholdValidator == nil {
|
|
t.Fatal("expected confidence_threshold validator")
|
|
}
|
|
|
|
cfg := config.Default()
|
|
cfg.Thresholds.Homophones = 0.95
|
|
result, err := thresholdValidator.Validate(context.Background(), validators.Request{
|
|
Config: &cfg,
|
|
ModuleKey: "homophones",
|
|
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
|
{
|
|
CorrectionProposal: proposals.CorrectionProposal{
|
|
TargetSegmentID: 1,
|
|
OriginalText: "gestures",
|
|
CorrectedText: "Jesters",
|
|
Confidence: 0.94,
|
|
},
|
|
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("validate error: %v", err)
|
|
}
|
|
if len(result.Decisions) != 1 {
|
|
t.Fatalf("expected one decision, got %+v", result.Decisions)
|
|
}
|
|
if result.Decisions[0].Approved {
|
|
t.Fatalf("expected rejection under homophones threshold, got %+v", result.Decisions[0])
|
|
}
|
|
if result.Decisions[0].ReasonCode != validators.ReasonLowConfidence {
|
|
t.Fatalf("expected low confidence reason, got %+v", result.Decisions[0])
|
|
}
|
|
}
|
|
|
|
func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testing.T) {
|
|
module, err := New()
|
|
if err != nil {
|
|
t.Fatalf("New error: %v", err)
|
|
}
|
|
var protectedValidator contracts.Validator
|
|
for _, v := range module.Validators() {
|
|
if v.Name() == "protected_terms" {
|
|
protectedValidator = v
|
|
break
|
|
}
|
|
}
|
|
if protectedValidator == nil {
|
|
t.Fatal("expected protected_terms validator")
|
|
}
|
|
|
|
working := &schema.Transcript{Segments: []schema.Segment{
|
|
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "Jesters are here."},
|
|
}}
|
|
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{
|
|
{Name: "Jesters", Category: "faction", Summary: "Faction"},
|
|
}}
|
|
result, err := protectedValidator.Validate(context.Background(), validators.Request{
|
|
WorkingTranscript: working,
|
|
Glossary: glossary,
|
|
ModuleKey: "homophones",
|
|
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
|
{
|
|
CorrectionProposal: proposals.CorrectionProposal{
|
|
TargetSegmentID: 1,
|
|
OriginalText: "Jesters",
|
|
CorrectedText: "gestures",
|
|
Confidence: 0.99,
|
|
},
|
|
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("validate error: %v", err)
|
|
}
|
|
if len(result.Decisions) != 1 {
|
|
t.Fatalf("expected one decision, got %+v", result.Decisions)
|
|
}
|
|
if result.Decisions[0].Approved {
|
|
t.Fatalf("expected protected-term rejection, got %+v", result.Decisions[0])
|
|
}
|
|
if result.Decisions[0].ReasonCode != validators.ReasonProtectedGlossaryTerm {
|
|
t.Fatalf("expected protected glossary-term reason, got %+v", result.Decisions[0])
|
|
}
|
|
}
|