Complete Phase 14 homophones module
This commit is contained in:
76
internal/modules/homophones/module.go
Normal file
76
internal/modules/homophones/module.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package homophones
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
type Module struct {
|
||||
validators []contracts.Validator
|
||||
}
|
||||
|
||||
func New() (*Module, error) {
|
||||
spokenForm, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Module{
|
||||
validators: []contracts.Validator{
|
||||
validators.NoEffectValidator{},
|
||||
validators.OriginalTextPresenceValidator{},
|
||||
validators.ConfidenceThresholdValidator{},
|
||||
validators.ProtectedGlossaryTermValidator{},
|
||||
validators.NonEmptyCorrectionValidator{},
|
||||
spokenForm,
|
||||
meaningReversal,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Module) Key() string { return "homophones" }
|
||||
|
||||
func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
// Python homophones module uses require_unique for conservative single-span replacement.
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
|
||||
func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
messages, err := BuildProposalMessages(req.WorkingTranscript, req.Glossary)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.WorkingTranscript,
|
||||
Section: req.Section,
|
||||
Glossary: req.Glossary,
|
||||
Config: req.Config,
|
||||
Messages: messages,
|
||||
StageName: fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName),
|
||||
StartIndex: 0,
|
||||
LLMClient: req.LLMClient,
|
||||
Scheduler: req.LLMScheduler,
|
||||
DiagnosticsDir: req.DiagnosticsDir,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return generated.Corrections, nil
|
||||
}
|
||||
296
internal/modules/homophones/module_test.go
Normal file
296
internal/modules/homophones/module_test.go
Normal file
@@ -0,0 +1,296 @@
|
||||
package homophones
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
responses []proposal_generation.StructuredCorrectionSet
|
||||
err error
|
||||
calls []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (f *fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
f.calls = append(f.calls, req)
|
||||
if f.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, f.err
|
||||
}
|
||||
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
|
||||
}
|
||||
if len(f.responses) == 0 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
|
||||
}
|
||||
*target = f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
type countingScheduler struct {
|
||||
runs int
|
||||
}
|
||||
|
||||
func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
||||
s.runs++
|
||||
return fn(ctx)
|
||||
}
|
||||
|
||||
func tinyTranscript() *schema.Transcript {
|
||||
return &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "There were gestures in the hall.", Categories: []string{"session"}},
|
||||
}}
|
||||
}
|
||||
|
||||
func tinyGlossary() *schema.Glossary {
|
||||
return &schema.Glossary{Entries: []schema.GlossaryEntry{
|
||||
{Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary())
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Protected glossary/context:",
|
||||
"Transcript section:",
|
||||
`"Aliases":`,
|
||||
`"Category": "faction"`,
|
||||
`"Summary": "Guild members"`,
|
||||
`"Plural": "Jesters"`,
|
||||
"homophone correction assistant",
|
||||
"mistranscriptions of spoken English",
|
||||
"Do not add or remove punctuation",
|
||||
"Do not return speaker, start, or end fields",
|
||||
`"id": 1`,
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{
|
||||
"summarize the transcript",
|
||||
"style rewrite",
|
||||
"grammar cleanup",
|
||||
"punctuation cleanup",
|
||||
} {
|
||||
if strings.Contains(strings.ToLower(combined), forbidden) {
|
||||
t.Fatalf("prompt should not invite %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomophonesModuleReplacementPolicy(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
}
|
||||
if m.ReplacementPolicy() != proposals.ReplacementPolicyRequireUnique {
|
||||
t.Fatalf("unexpected replacement policy: %q", m.ReplacementPolicy())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomophonesModuleValidatorChain(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
}
|
||||
got := make([]string, 0, len(m.Validators()))
|
||||
for _, v := range m.Validators() {
|
||||
got = append(got, v.Name())
|
||||
}
|
||||
want := []string{
|
||||
"no_effect",
|
||||
"original_text_presence",
|
||||
"confidence_threshold",
|
||||
"protected_glossary_terms",
|
||||
"non_empty_correction",
|
||||
"spoken_form_plausibility_review",
|
||||
"meaning_reversal_review",
|
||||
}
|
||||
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("unexpected validator chain\n got: %v\nwant: %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
|
||||
secret := "homophones-secret"
|
||||
client := &fakeLLMClient{
|
||||
responses: []proposal_generation.StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
}
|
||||
diagDir := t.TempDir()
|
||||
out, err := m.Propose(context.Background(), contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
Config: &cfg,
|
||||
WorkingTranscript: tinyTranscript(),
|
||||
Glossary: tinyGlossary(),
|
||||
DiagnosticsDir: diagDir,
|
||||
},
|
||||
RunSpec: contracts.ModuleRunSpec{
|
||||
ModuleKey: "homophones",
|
||||
InstanceName: "homophones",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
},
|
||||
LLMClient: client,
|
||||
LLMScheduler: scheduler,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Propose error: %v", err)
|
||||
}
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" {
|
||||
t.Fatalf("expected one homophones:proposal call, got %+v", client.calls)
|
||||
}
|
||||
if len(out) != 1 || out[0].CorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json"))
|
||||
if globErr != nil {
|
||||
t.Fatalf("glob error: %v", globErr)
|
||||
}
|
||||
if len(diagFiles) == 0 {
|
||||
t.Fatalf("expected diagnostics response payload under %s", filepath.Join(diagDir, "homophones"))
|
||||
}
|
||||
for _, f := range diagFiles {
|
||||
raw, readErr := os.ReadFile(f)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read diag %q: %v", f, readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomophonesConfidenceThresholdIsUsed(t *testing.T) {
|
||||
module, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
}
|
||||
var thresholdValidator contracts.Validator
|
||||
for _, v := range module.Validators() {
|
||||
if v.Name() == "confidence_threshold" {
|
||||
thresholdValidator = v
|
||||
break
|
||||
}
|
||||
}
|
||||
if thresholdValidator == nil {
|
||||
t.Fatal("expected confidence_threshold validator")
|
||||
}
|
||||
|
||||
cfg := config.Default()
|
||||
cfg.Thresholds.Homophones = 0.95
|
||||
result, err := thresholdValidator.Validate(context.Background(), validators.Request{
|
||||
Config: &cfg,
|
||||
ModuleKey: "homophones",
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 0.94,
|
||||
},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("validate error: %v", err)
|
||||
}
|
||||
if len(result.Decisions) != 1 {
|
||||
t.Fatalf("expected one decision, got %+v", result.Decisions)
|
||||
}
|
||||
if result.Decisions[0].Approved {
|
||||
t.Fatalf("expected rejection under homophones threshold, got %+v", result.Decisions[0])
|
||||
}
|
||||
if result.Decisions[0].ReasonCode != validators.ReasonLowConfidence {
|
||||
t.Fatalf("expected low confidence reason, got %+v", result.Decisions[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testing.T) {
|
||||
module, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
}
|
||||
var protectedValidator contracts.Validator
|
||||
for _, v := range module.Validators() {
|
||||
if v.Name() == "protected_glossary_terms" {
|
||||
protectedValidator = v
|
||||
break
|
||||
}
|
||||
}
|
||||
if protectedValidator == nil {
|
||||
t.Fatal("expected protected_glossary_terms validator")
|
||||
}
|
||||
|
||||
working := &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "Jesters are here."},
|
||||
}}
|
||||
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{
|
||||
{Name: "Jesters", Category: "faction", Summary: "Faction"},
|
||||
}}
|
||||
result, err := protectedValidator.Validate(context.Background(), validators.Request{
|
||||
WorkingTranscript: working,
|
||||
Glossary: glossary,
|
||||
ModuleKey: "homophones",
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "Jesters",
|
||||
CorrectedText: "gestures",
|
||||
Confidence: 0.99,
|
||||
},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("validate error: %v", err)
|
||||
}
|
||||
if len(result.Decisions) != 1 {
|
||||
t.Fatalf("expected one decision, got %+v", result.Decisions)
|
||||
}
|
||||
if result.Decisions[0].Approved {
|
||||
t.Fatalf("expected protected-term rejection, got %+v", result.Decisions[0])
|
||||
}
|
||||
if result.Decisions[0].ReasonCode != validators.ReasonProtectedGlossaryTerm {
|
||||
t.Fatalf("expected protected glossary-term reason, got %+v", result.Decisions[0])
|
||||
}
|
||||
}
|
||||
83
internal/modules/homophones/prompt.go
Normal file
83
internal/modules/homophones/prompt.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package homophones
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type promptSegment struct {
|
||||
ID int `json:"id"`
|
||||
Speaker string `json:"speaker"`
|
||||
Start float64 `json:"start"`
|
||||
End float64 `json:"end"`
|
||||
Text string `json:"text"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
}
|
||||
|
||||
type promptTranscriptSection struct {
|
||||
SectionIndex int `json:"section_index"`
|
||||
Segments []promptSegment `json:"segments"`
|
||||
}
|
||||
|
||||
// BuildProposalMessages mirrors the Python homophones-module prompt intent:
|
||||
// conservative homophone and mistranscription correction only.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary) ([]contracts.LLMMessage, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionPayload := promptTranscriptSection{
|
||||
SectionIndex: 0,
|
||||
Segments: make([]promptSegment, 0),
|
||||
}
|
||||
if transcript != nil {
|
||||
for _, s := range transcript.Segments {
|
||||
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
|
||||
ID: s.ID,
|
||||
Speaker: s.Speaker,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
Text: s.Text,
|
||||
Categories: append([]string(nil), s.Categories...),
|
||||
})
|
||||
}
|
||||
}
|
||||
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
|
||||
system := "You are Audita, a conservative homophone correction assistant. Identify only transcript changes that plausibly reflect homophones, phonetic similarity, or common mistranscriptions of spoken English. Do not make punctuation, capitalization, spacing, filler-word, repetition, style, or grammar edits. Do not paraphrase, summarize, or rewrite content."
|
||||
user := "Review this transcript section and return only homophone or spoken-form corrections that should be applied.\n\n" +
|
||||
"Rules:\n" +
|
||||
"- Approve only corrections where the original text is plausibly a mistaken homophone, phonetic rendering, or mistranscription of what was likely spoken.\n" +
|
||||
"- Allow examples such as changing \"dam\" to \"damn\", \"rank\" to \"Hrank\", or \"gestures\" to \"Jesters\" when local context supports the correction.\n" +
|
||||
"- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n" +
|
||||
"- Reject antonyms or reversals such as changing \"visible\" to \"invisible\".\n" +
|
||||
"- Do not add or remove punctuation, alter capitalization only, normalize spacing, remove filler words, collapse repetitions, or make general readability edits.\n" +
|
||||
"- Treat glossary names and aliases as protected spellings and context.\n" +
|
||||
"- You may correct toward glossary names, aliases, or their plural forms when the correction is acoustically plausible and supported by local context.\n" +
|
||||
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n" +
|
||||
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" +
|
||||
"- If a segment includes categories, treat them as additional transcript context.\n" +
|
||||
"- Use the exact id from the input segment.\n" +
|
||||
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" +
|
||||
"- Choose an original_text span that appears exactly once in the current segment text.\n" +
|
||||
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" +
|
||||
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" +
|
||||
"- Do not return corrections where original_text and corrected_text are identical.\n" +
|
||||
"- Do not return speaker, start, or end fields.\n" +
|
||||
"- Return only changed segments; do not return entries for unchanged segments.\n" +
|
||||
"- confidence must be between 0.0 and 1.0.\n" +
|
||||
"- If no corrections are needed, return an empty corrections list.\n\n" +
|
||||
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
|
||||
|
||||
return []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user