Complete Phase 14 homophones module

This commit is contained in:
2026-05-12 11:37:35 +00:00
parent 543a7ff8ef
commit dbf3605712
10 changed files with 858 additions and 57 deletions

View File

@@ -460,7 +460,7 @@ func extractErrorPhase(err error) (phase string, message string) {
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
report := reporting.ProcessReport{
Phase: "phase13-glossary-module",
Phase: "phase14-homophones-module",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,

View File

@@ -616,8 +616,8 @@ func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) {
if report.Chunking.MaxSectionTokens == 0 {
t.Errorf("expected max_section_tokens in report")
}
if report.Phase != "phase13-glossary-module" {
t.Errorf("expected phase 'phase13-glossary-module', got %q", report.Phase)
if report.Phase != "phase14-homophones-module" {
t.Errorf("expected phase 'phase14-homophones-module', got %q", report.Phase)
}
}
@@ -924,7 +924,7 @@ func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "homophones",
"--modules", "spoken_word",
"--work-dir", workDir,
"--work-dir-retention", "always",
"--report-json", reportPath,
@@ -955,7 +955,7 @@ func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.
}
func TestRunProcessExplicitUnimplementedModulesFailClearly(t *testing.T) {
for _, moduleKey := range []string{"homophones", "spoken_word"} {
for _, moduleKey := range []string{"spoken_word"} {
t.Run(moduleKey, func(t *testing.T) {
var stdout, stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
@@ -1472,6 +1472,324 @@ func TestRunProcessExplicitGlossaryMalformedLLMOutputFailsWithErrorLog(t *testin
}
}
func TestRunProcessExplicitHomophonesAppliesCorrectionAndReportsDiagnostics(t *testing.T) {
secret := "phase14-secret"
proposalClient := &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.95},
},
},
},
}
validationClient := &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "spoken plausible"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: secret}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Setenv("AUDITA_LLM_API_KEY", secret)
t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", secret)
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"There were gestures in the hall."}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "homophones",
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
"--work-dir-retention", "always",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "There were Jesters in the hall." {
t.Fatalf("expected homophones correction applied, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 || report.ModuleResults[0].ModuleKey != "homophones" {
t.Fatalf("expected one homophones module result, got %+v", report.ModuleResults)
}
if len(report.ModuleResults[0].AppliedChanges) != 1 {
t.Fatalf("expected one applied homophones change, got %+v", report.ModuleResults[0].AppliedChanges)
}
if len(report.ModuleResults[0].ValidatorDecisions) == 0 {
t.Fatalf("expected validator decisions in report")
}
runDir := onlyRunDir(t, workDir)
runReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
if len(runReport.ModuleResults) != 1 || runReport.ModuleResults[0].ModuleKey != "homophones" {
t.Fatalf("expected homophones module results in run-dir report, got %+v", runReport.ModuleResults)
}
diagFiles, globErr := filepath.Glob(filepath.Join(runDir, "homophones", "*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
}
if len(diagFiles) == 0 {
t.Fatalf("expected homophones diagnostics payload files in %s", filepath.Join(runDir, "homophones"))
}
for _, f := range diagFiles {
raw := string(readFile(t, f))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in diagnostics %q: %s", f, raw)
}
}
}
func TestRunProcessExplicitHomophonesRejectedAndApplicationSkipAreDistinct(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hall", CorrectedText: "temple", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "there were gestures in the hall", CorrectedText: "there were gestures in the temple", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "and there were gestures.", CorrectedText: "and there were jesters.", Confidence: 0.99},
},
},
},
}
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.9, Reason: "reject"},
},
},
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
},
},
},
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"there were gestures in the hall and there were gestures."}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "homophones",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 {
t.Fatalf("expected one module result")
}
module := report.ModuleResults[0]
if len(module.ValidatorRejected) != 1 {
t.Fatalf("expected one validator rejection, got %+v", module.ValidatorRejected)
}
if len(module.SkippedChanges) != 1 {
t.Fatalf("expected one application skip, got %+v", module.SkippedChanges)
}
if module.ValidatorRejected[0].ReasonCode == string(module.SkippedChanges[0].SkipReason) {
t.Fatalf("validator rejection and application skip should remain distinct")
}
}
func TestRunProcessExplicitHomophonesProtectedGlossaryTermRejected(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Audita", CorrectedText: "audita", Confidence: 0.99},
},
},
},
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Audita held the line."}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "homophones",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "Audita held the line." {
t.Fatalf("expected protected glossary term to remain unchanged, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 {
t.Fatalf("expected one module result, got %+v", report.ModuleResults)
}
module := report.ModuleResults[0]
if len(module.AppliedChanges) != 0 {
t.Fatalf("expected no applied changes, got %+v", module.AppliedChanges)
}
if len(module.ValidatorRejected) != 1 {
t.Fatalf("expected one validator rejection, got %+v", module.ValidatorRejected)
}
if module.ValidatorRejected[0].ReasonCode != validators.ReasonProtectedGlossaryTerm {
t.Fatalf("expected protected glossary term rejection, got %+v", module.ValidatorRejected[0])
}
}
func TestRunProcessExplicitHomophonesMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "homophones",
"--work-dir", workDir,
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed homophones run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
}
}
func TestRunProcessExplicitGlossaryThenHomophonesSeesWorkingTranscriptChanges(t *testing.T) {
proposalClient := &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.99},
},
},
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "Jester", Confidence: 0.99},
},
},
},
}
validationClient := &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"There were gestures in the hall."}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "glossary,homophones",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "There were Jester in the hall." {
t.Fatalf("expected homophones stage to see glossary output, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 2 {
t.Fatalf("expected two module results, got %+v", report.ModuleResults)
}
if report.ModuleResults[0].ModuleKey != "glossary" || report.ModuleResults[1].ModuleKey != "homophones" {
t.Fatalf("expected glossary then homophones results, got %+v", report.ModuleResults)
}
if len(report.ModuleResults[0].AppliedChanges) != 1 || len(report.ModuleResults[1].AppliedChanges) != 1 {
t.Fatalf("expected one applied change per module, got %+v", report.ModuleResults)
}
}
func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer

View File

@@ -11,7 +11,7 @@ import (
func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) {
now := time.Now().UTC()
report := ProcessReport{
Phase: "phase13-glossary-module",
Phase: "phase14-homophones-module",
Status: "success",
ModuleResults: []ModuleReport{
{
@@ -80,7 +80,7 @@ func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) {
func TestProcessReportModuleResultsJSONFailedModule(t *testing.T) {
now := time.Now().UTC()
report := ProcessReport{
Phase: "phase13-glossary-module",
Phase: "phase14-homophones-module",
Status: "failed",
ModuleResults: []ModuleReport{
{

View File

@@ -10,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
glossarymodule "gitea.maximumdirect.net/eric/audita/internal/modules/glossary"
grammarmodule "gitea.maximumdirect.net/eric/audita/internal/modules/grammar"
homophonesmodule "gitea.maximumdirect.net/eric/audita/internal/modules/homophones"
)
const (
@@ -72,6 +73,7 @@ func NewFactory(deps Dependencies) *Factory {
constructors: make(map[string]Constructor, len(knownModuleKeys)),
}
_ = factory.RegisterConstructor(ModuleKeyGlossary, constructGlossaryModule)
_ = factory.RegisterConstructor(ModuleKeyHomophones, constructHomophonesModule)
_ = factory.RegisterConstructor(ModuleKeyGrammar, constructGrammarModule)
return factory
}
@@ -104,6 +106,12 @@ func constructGlossaryModule(ctx context.Context, req ConstructRequest) (contrac
return glossarymodule.New()
}
func constructHomophonesModule(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {
_ = ctx
_ = req
return homophonesmodule.New()
}
// ModuleForSpec resolves one configured run spec into a module instance.
func (f *Factory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
if f == nil {

View File

@@ -66,7 +66,7 @@ func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) {
func TestRecognizedButUnimplementedModuleKeyFailsCleanly(t *testing.T) {
factory := NewFactory(Dependencies{})
for _, key := range []string{ModuleKeyHomophones, ModuleKeySpokenWord} {
for _, key := range []string{ModuleKeySpokenWord} {
t.Run(key, func(t *testing.T) {
_, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: key, InstanceName: key})
if err == nil {
@@ -106,6 +106,17 @@ func TestGlossaryIsRegisteredAndConstructibleByDefault(t *testing.T) {
}
}
func TestHomophonesIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyHomophones, InstanceName: ModuleKeyHomophones})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
}
if module.Key() != ModuleKeyHomophones {
t.Fatalf("expected homophones module key, got %q", module.Key())
}
}
func TestRegisterConstructorAndConstruct(t *testing.T) {
factory := NewFactory(Dependencies{})
if err := factory.RegisterConstructor(ModuleKeyGlossary, func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {

View File

@@ -0,0 +1,76 @@
package homophones
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type Module struct {
validators []contracts.Validator
}
func New() (*Module, error) {
spokenForm, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility, "")
if err != nil {
return nil, err
}
meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "")
if err != nil {
return nil, err
}
return &Module{
validators: []contracts.Validator{
validators.NoEffectValidator{},
validators.OriginalTextPresenceValidator{},
validators.ConfidenceThresholdValidator{},
validators.ProtectedGlossaryTermValidator{},
validators.NonEmptyCorrectionValidator{},
spokenForm,
meaningReversal,
},
}, nil
}
func (m *Module) Key() string { return "homophones" }
func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
// Python homophones module uses require_unique for conservative single-span replacement.
return proposals.ReplacementPolicyRequireUnique
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
messages, err := BuildProposalMessages(req.WorkingTranscript, req.Glossary)
if err != nil {
return nil, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Section: req.Section,
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
StageName: fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return nil, err
}
return generated.Corrections, nil
}

View File

@@ -0,0 +1,296 @@
package homophones
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type fakeLLMClient struct {
responses []proposal_generation.StructuredCorrectionSet
err error
calls []contracts.StructuredCompletionRequest
}
func (f *fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
f.calls = append(f.calls, req)
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
}
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
}
*target = f.responses[0]
f.responses = f.responses[1:]
return contracts.StructuredCompletionResponse{}, nil
}
type countingScheduler struct {
runs int
}
func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
s.runs++
return fn(ctx)
}
func tinyTranscript() *schema.Transcript {
return &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "There were gestures in the hall.", Categories: []string{"session"}},
}}
}
func tinyGlossary() *schema.Glossary {
return &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"},
}}
}
func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary())
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
if len(msgs) != 2 {
t.Fatalf("expected 2 messages, got %d", len(msgs))
}
combined := msgs[0].Content + "\n" + msgs[1].Content
for _, want := range []string{
"Protected glossary/context:",
"Transcript section:",
`"Aliases":`,
`"Category": "faction"`,
`"Summary": "Guild members"`,
`"Plural": "Jesters"`,
"homophone correction assistant",
"mistranscriptions of spoken English",
"Do not add or remove punctuation",
"Do not return speaker, start, or end fields",
`"id": 1`,
} {
if !strings.Contains(combined, want) {
t.Fatalf("expected prompt to contain %q", want)
}
}
for _, forbidden := range []string{
"summarize the transcript",
"style rewrite",
"grammar cleanup",
"punctuation cleanup",
} {
if strings.Contains(strings.ToLower(combined), forbidden) {
t.Fatalf("prompt should not invite %q", forbidden)
}
}
}
func TestHomophonesModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
if m.ReplacementPolicy() != proposals.ReplacementPolicyRequireUnique {
t.Fatalf("unexpected replacement policy: %q", m.ReplacementPolicy())
}
}
func TestHomophonesModuleValidatorChain(t *testing.T) {
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
got := make([]string, 0, len(m.Validators()))
for _, v := range m.Validators() {
got = append(got, v.Name())
}
want := []string{
"no_effect",
"original_text_presence",
"confidence_threshold",
"protected_glossary_terms",
"non_empty_correction",
"spoken_form_plausibility_review",
"meaning_reversal_review",
}
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("unexpected validator chain\n got: %v\nwant: %v", got, want)
}
}
func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
secret := "homophones-secret"
client := &fakeLLMClient{
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.94},
},
},
},
}
scheduler := &countingScheduler{}
cfg := config.Default()
cfg.PrimaryLLM.APIKey = secret
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
diagDir := t.TempDir()
out, err := m.Propose(context.Background(), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: "homophones",
InstanceName: "homophones",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
},
LLMClient: client,
LLMScheduler: scheduler,
})
if err != nil {
t.Fatalf("Propose error: %v", err)
}
if scheduler.runs != 1 {
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
}
if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" {
t.Fatalf("expected one homophones:proposal call, got %+v", client.calls)
}
if len(out) != 1 || out[0].CorrectedText != "Jesters" {
t.Fatalf("unexpected proposals: %+v", out)
}
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "homophones", "*proposal*response-payload.json"))
if globErr != nil {
t.Fatalf("glob error: %v", globErr)
}
if len(diagFiles) == 0 {
t.Fatalf("expected diagnostics response payload under %s", filepath.Join(diagDir, "homophones"))
}
for _, f := range diagFiles {
raw, readErr := os.ReadFile(f)
if readErr != nil {
t.Fatalf("read diag %q: %v", f, readErr)
}
if strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
}
}
}
func TestHomophonesConfidenceThresholdIsUsed(t *testing.T) {
module, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
var thresholdValidator contracts.Validator
for _, v := range module.Validators() {
if v.Name() == "confidence_threshold" {
thresholdValidator = v
break
}
}
if thresholdValidator == nil {
t.Fatal("expected confidence_threshold validator")
}
cfg := config.Default()
cfg.Thresholds.Homophones = 0.95
result, err := thresholdValidator.Validate(context.Background(), validators.Request{
Config: &cfg,
ModuleKey: "homophones",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: 1,
OriginalText: "gestures",
CorrectedText: "Jesters",
Confidence: 0.94,
},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
},
})
if err != nil {
t.Fatalf("validate error: %v", err)
}
if len(result.Decisions) != 1 {
t.Fatalf("expected one decision, got %+v", result.Decisions)
}
if result.Decisions[0].Approved {
t.Fatalf("expected rejection under homophones threshold, got %+v", result.Decisions[0])
}
if result.Decisions[0].ReasonCode != validators.ReasonLowConfidence {
t.Fatalf("expected low confidence reason, got %+v", result.Decisions[0])
}
}
func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testing.T) {
module, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
var protectedValidator contracts.Validator
for _, v := range module.Validators() {
if v.Name() == "protected_glossary_terms" {
protectedValidator = v
break
}
}
if protectedValidator == nil {
t.Fatal("expected protected_glossary_terms validator")
}
working := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "Jesters are here."},
}}
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Category: "faction", Summary: "Faction"},
}}
result, err := protectedValidator.Validate(context.Background(), validators.Request{
WorkingTranscript: working,
Glossary: glossary,
ModuleKey: "homophones",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: 1,
OriginalText: "Jesters",
CorrectedText: "gestures",
Confidence: 0.99,
},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
},
})
if err != nil {
t.Fatalf("validate error: %v", err)
}
if len(result.Decisions) != 1 {
t.Fatalf("expected one decision, got %+v", result.Decisions)
}
if result.Decisions[0].Approved {
t.Fatalf("expected protected-term rejection, got %+v", result.Decisions[0])
}
if result.Decisions[0].ReasonCode != validators.ReasonProtectedGlossaryTerm {
t.Fatalf("expected protected glossary-term reason, got %+v", result.Decisions[0])
}
}

View File

@@ -0,0 +1,83 @@
package homophones
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
)
type promptSegment struct {
ID int `json:"id"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}
type promptTranscriptSection struct {
SectionIndex int `json:"section_index"`
Segments []promptSegment `json:"segments"`
}
// BuildProposalMessages mirrors the Python homophones-module prompt intent:
// conservative homophone and mistranscription correction only.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
}
sectionPayload := promptTranscriptSection{
SectionIndex: 0,
Segments: make([]promptSegment, 0),
}
if transcript != nil {
for _, s := range transcript.Segments {
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: append([]string(nil), s.Categories...),
})
}
}
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
}
system := "You are Audita, a conservative homophone correction assistant. Identify only transcript changes that plausibly reflect homophones, phonetic similarity, or common mistranscriptions of spoken English. Do not make punctuation, capitalization, spacing, filler-word, repetition, style, or grammar edits. Do not paraphrase, summarize, or rewrite content."
user := "Review this transcript section and return only homophone or spoken-form corrections that should be applied.\n\n" +
"Rules:\n" +
"- Approve only corrections where the original text is plausibly a mistaken homophone, phonetic rendering, or mistranscription of what was likely spoken.\n" +
"- Allow examples such as changing \"dam\" to \"damn\", \"rank\" to \"Hrank\", or \"gestures\" to \"Jesters\" when local context supports the correction.\n" +
"- Reject unrelated substitutions like changing \"Lyra\" to \"Jesters\".\n" +
"- Reject antonyms or reversals such as changing \"visible\" to \"invisible\".\n" +
"- Do not add or remove punctuation, alter capitalization only, normalize spacing, remove filler words, collapse repetitions, or make general readability edits.\n" +
"- Treat glossary names and aliases as protected spellings and context.\n" +
"- You may correct toward glossary names, aliases, or their plural forms when the correction is acoustically plausible and supported by local context.\n" +
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n" +
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" +
"- If a segment includes categories, treat them as additional transcript context.\n" +
"- Use the exact id from the input segment.\n" +
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" +
"- Choose an original_text span that appears exactly once in the current segment text.\n" +
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" +
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" +
"- Do not return corrections where original_text and corrected_text are identical.\n" +
"- Do not return speaker, start, or end fields.\n" +
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{
{Role: "system", Content: system},
{Role: "user", Content: user},
}, nil
}